Skip to content
This repository was archived by the owner on Aug 4, 2022. It is now read-only.

Next (WIP) #69

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Next (WIP) #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88f0ceb
feat(nlp.js): upgrade to v4 & improve logging
eric-lara Feb 1, 2020
7937159
feat(train): in foreground (wip)
eric-lara Feb 2, 2020
c15a128
chore: improve logs
eric-lara Feb 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7,995 changes: 3,620 additions & 4,375 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "nlpjs-app",
"version": "1.4.7",
"version": "1.5.0",
"description": "Application to help you train your agents for bots, done using NLP.js.",
"main": "./server/index.js",
"scripts": {
Expand Down Expand Up @@ -30,7 +30,7 @@
"url": "https://github.com/axa-group/nlp.js-app.git"
},
"dependencies": {
"@hapi/hapi": "^18.4.0",
"@hapi/hapi": "^18.4.1",
"braces": "^2.3.2",
"dotenv": "^6.1.0",
"hapi-swagger": "^9.1.2",
Expand All @@ -40,7 +40,7 @@
"joigoose": "^4.0.8",
"lodash": "^4.17.15",
"mongoose": "^5.8.7",
"node-nlp": "^3.10.2",
"node-nlp": "^4.2.0",
"pino": "^5.13.4",
"pino-pretty": "^3.2.2",
"vision": "^5.4.4"
Expand Down
2 changes: 1 addition & 1 deletion public/3.d974938a535e02b271f5.chunk.js → public/3.fc58d9aa5ba76bbcfa6d.chunk.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion public/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="manifest" href="manifest.json"><meta name="mobile-web-app-capable" content="yes"><title>NLP.js</title><script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.min.css"><script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script><link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700" rel="stylesheet"><link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet"><!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.js"></script>
<![endif]--></head><body><noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript><div id="app"></div><script type="text/javascript" src="/main.702632cb92fd2103d25d.js"></script></body></html>
<![endif]--></head><body><noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript><div id="app"></div><script type="text/javascript" src="/main.4a40d5e68e3eeb1a3f47.js"></script></body></html>
2 changes: 1 addition & 1 deletion public/main.702632cb92fd2103d25d.js → public/main.4a40d5e68e3eeb1a3f47.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion public/sw.js
View file
Open in desktop

Large diffs are not rendered by default.

34 changes: 25 additions & 9 deletions server/feats/agent/agent.controllers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,15 +353,22 @@ async function train(request) {
const agentId = request.params.id;

if (!app.existsTraining(agentId)) {
logger.debug(`training of agent ${agentId} is not in memory, getting training from db...`);
const training = await app.database.findOne(Model.Training, {
'any.agentId': agentId
});
if (training) {
const model = JSON.parse(training.any.model);
app.loadTraining(agentId, model);
} else {
logger.debug('there\'s no training in db');
}
} else {
logger.debug(`reusing training from agent ${agentId} (already in memory)`);
}

const agent = await app.database.findById(Model.Agent, agentId);

if (!agent) {
return app.error(404, 'The agent was not found');
}
Expand All @@ -377,22 +384,31 @@ async function train(request) {
scenarios,
entities
};

if (agent.status !== AgentStatus.Training) {
agent.status = AgentStatus.Training;
app.database.saveItem(agent);
let model = await app.train(data);
if (model) {
await app.database.deleteMany(Model.Training, { 'any.agentId': agentId });
model = JSON.stringify(model);
await app.database.save(Model.Training, { any: { agentId, model } });
}
agent.lastTraining = new Date();
agent.status = AgentStatus.Ready;
return app.database.saveItem(agent);

await trainAsync(data, agent, agentId);
}
return {};
}

async function trainAsync(data, agent, agentId) {
logger.debug(`starting async training...`);
let model = await app.train(data);
logger.debug(`end training`);
if (model) {
await app.database.deleteMany(Model.Training, { 'any.agentId': agentId });
model = JSON.stringify(model);
logger.debug(`replacing training in db`);
await app.database.save(Model.Training, { any: { agentId, model } });
}
agent.lastTraining = new Date();
agent.status = AgentStatus.Ready;
return app.database.saveItem(agent);
}

/**
* Method that performs replacements of entity values in the answer
* @param answer incoming from nlp.js result)
Expand Down
4 changes: 3 additions & 1 deletion server/models/intent.model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ module.exports = {
domain: Joi.string().trim(),
domainName: Joi.string().trim(),
intentName: Joi.string().trim(),
actionDescription: Joi.string().optional().trim(),
actionDescription: Joi.string()
.optional()
.trim(),
examples: Joi.array().items(ExampleIntentModel),
useWebhook: Joi.boolean(),
usePostFormat: Joi.boolean()
Expand Down
32 changes: 27 additions & 5 deletions server/trainers/nlpjs-trainer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@

const childProcess = require('child_process');
const { NlpManager } = require('node-nlp');

const { useNeuralSettings } = require('./nlpjs-settings');
const Logger = require('../common/logger');

const logger = Logger.getInstance();

/**
* Class for a NLP.JS trainer
Expand All @@ -42,9 +46,11 @@ class NlpjsTrainer {
* @param {object} data Training data.
*/
addEntities(manager, data) {
logger.debug('adding entities...');
data.entities.forEach(entity => {
const { entityName } = entity;
if (entity.type === 'enum') {
logger.debug(`adding named entity ${entityName}, ${entity.examples.length} examples`);
for (let i = 0; i < entity.examples.length; i += 1) {
const example = entity.examples[i];
const optionName = example.value;
Expand All @@ -56,6 +62,7 @@ class NlpjsTrainer {
}
} else if (entity.type === 'regex') {
const languages = entity.languages || manager.settings.languages;
logger.debug(`adding regex entity ${entityName} (${languages})`);
manager.addRegexEntity(entityName, languages, entity.regex);
}
});
Expand Down Expand Up @@ -112,14 +119,17 @@ class NlpjsTrainer {
* @param {object} data Training data.
*/
addIntents(manager, data) {
logger.debug(`adding intents... ${data.intents.length}`);
data.intents.forEach(intent => {
const domain = this.getDomain(intent.domain, data);
const language = domain.language || manager.settings.languages[0];
const { intentName } = intent;
manager.assignDomain(domain.language || manager.settings.languages[0], intentName, domain.domainName);
manager.assignDomain(language, intentName, domain.domainName);

logger.trace(`assigning intent ${intentName} (${language}) to domain ${domain.domainName}`);

for (let i = 0; i < intent.examples.length; i += 1) {
const example = intent.examples[i];
const language = domain.language || manager.settings.languages[0];
const utterance = example.userSays;
manager.addDocument(language, utterance, intentName);
}
Expand All @@ -132,11 +142,13 @@ class NlpjsTrainer {
* @param {object} data Training data.
*/
addAnswers(manager, data) {
logger.debug(`adding answers...`);
data.scenarios.forEach(scenario => {
const domain = this.getDomain(scenario.domain, data);
const language = domain.language || manager.settings.languages[0];
const intentName = this.getIntentName(scenario.intent, data);

logger.trace(`adding answers (${scenario.intentResponses.length}) to intent ${intentName} (${language})`);
for (let i = 0; i < scenario.intentResponses.length; i += 1) {
const answer = scenario.intentResponses[i];
manager.addAnswer(language, intentName, answer);
Expand All @@ -150,23 +162,27 @@ class NlpjsTrainer {
* @param {object} data Training data.
*/
addSlots(manager, data) {
logger.debug(`adding slots...`);
data.scenarios.forEach(scenario => {
const domain = this.getDomain(scenario.domain, data);
const language = domain.language || manager.settings.languages[0];
const intentName = this.getIntentName(scenario.intent, data);
if (scenario.slots && scenario.slots.length > 0) {
scenario.slots.forEach(slot => {
if (slot.isRequired) {
const managerSlot = manager.slotManager.getSlot(intentName, slot.entity);
const slotManager = manager.container.get('SlotManager');
const managerSlot = slotManager.getSlot(intentName, slot.entity);
if (managerSlot) {
logger.debug(`using slot for intent ${intentName} for entity ${slot.entity}`);
const texts = managerSlot.locales;
const text = slot.textPrompts[0];
texts[language] = text;
} else {
const texts = {};
const text = slot.textPrompts[0];
texts[language] = text;
manager.slotManager.addSlot(intentName, slot.entity, true, texts);
logger.debug(`adding slot to intent ${intentName} for entity ${slot.entity}`);
slotManager.addSlot(intentName, slot.entity, true, texts);
}
}
});
Expand Down Expand Up @@ -209,12 +225,17 @@ class NlpjsTrainer {
});
// eslint-disable-next-line no-underscore-dangle
this.managers[data.agent._id] = manager;

this.addEntities(manager, data);
this.addIntents(manager, data);
this.addAnswers(manager, data);
this.addSlots(manager, data);
const result = await this.trainProcess(manager.export());

const json = manager.export();
const result = await this.trainProcess(json);

manager.import(result);

return result;
}

Expand All @@ -239,6 +260,7 @@ class NlpjsTrainer {
if (model.nerManager && !model.nerManager.namedEntities) {
model.nerManager.namedEntities = {};
}
logger.debug(`importing model in manager ${agentId}`);
this.managers[agentId].import(model);
}

Expand Down