Merge pull request #552 from codeforequity-at/develop

Botium Core 1.9.10

index.js

@@ -9,5 +9,6 @@ module.exports = {
   ScriptingMemory: require('./src/scripting/ScriptingMemory'),
   Lib: {
     tryLoadPlugin: require('./src/containers/plugins/index').tryLoadPlugin
-  }
+  },
+  BotiumMockRichMessageTypes: require('./src/mocks/BotiumMockRichMessageTypes')
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.9.9",
+  "version": "1.9.10",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -16,7 +16,7 @@
     "agent": "node ./src/grid/agent/agent.js",
     "agent-jsdoc": "swagger-jsdoc -d ./src/grid/agent/swaggerDef.json -o ./src/grid/agent/swagger.json ./src/grid/agent/routes.js",
     "link": "npm link botium-connector-dialogflow botium-connector-webdriverio botium-connector-directline3 botium-connector-watson botium-connector-alexa-smapi botium-connector-echo",
-    "test": "NODE_PATH=\"./test/plugins/plugindir/fromfolder:./test/plugins/plugindir/fromfile\" mocha \"./test/**/*.spec.js\"",
+    "test": "cross-env NODE_PATH=\"./test/plugins/plugindir/fromfolder:./test/plugins/plugindir/fromfile:./test/security/resources\" mocha \"./test/**/*.spec.js\"",
     "coverage:report": "nyc report --reporter=lcov npm test",
     "license-checker": "license-checker > LICENSES-3RDPARTY.txt"
   },
@@ -36,41 +36,41 @@
     "body-parser": "^1.19.0",
     "boolean": "^3.0.1",
     "bottleneck": "^2.19.5",
-    "copy-dir": "^1.2.0",
+    "copy-dir": "^1.3.0",
     "csv-parse": "^4.9.0",
     "debug": "^4.1.1",
     "esprima": "^4.0.1",
     "express": "^4.17.1",
     "find-root": "^1.1.0",
     "glob": "^7.1.6",
-    "ioredis": "^4.16.3",
+    "ioredis": "^4.17.3",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.0.2",
-    "lodash": "^4.17.15",
-    "markdown-it": "^10.0.0",
+    "lodash": "^4.17.19",
+    "markdown-it": "^11.0.0",
     "mime-types": "^2.1.27",
     "mkdirp": "^1.0.4",
-    "moment": "^2.24.0",
+    "moment": "^2.27.0",
     "mustache": "^4.0.1",
-    "promise-retry": "^1.1.1",
+    "promise-retry": "^2.0.1",
     "promise.allsettled": "^1.0.2",
     "randomatic": "^3.1.1",
     "request": "^2.88.2",
     "rimraf": "^3.0.2",
     "sanitize-filename": "^1.6.3",
-    "slugify": "^1.4.0",
+    "slugify": "^1.4.5",
     "socket.io": "^2.3.0",
     "socket.io-client": "^2.3.0",
     "socketio-auth": "^0.1.1",
-    "swagger-jsdoc": "^3.5.0",
+    "swagger-jsdoc": "^4.0.0",
     "swagger-ui-express": "^4.1.4",
     "tcp-port-used": "^1.0.1",
-    "uuid": "^7.0.3",
+    "uuid": "^8.3.0",
     "write-yaml": "^1.0.0",
-    "xlsx": "^0.15.6",
+    "xlsx": "^0.16.6",
     "xregexp": "^4.3.0",
-    "yaml": "^1.9.2"
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
     "@babel/core": "^7.9.0",

src/Capabilities.js

@@ -131,6 +131,9 @@ module.exports = {
   SCRIPTING_UTTEXPANSION_RANDOM_COUNT: 'SCRIPTING_UTTEXPANSION_RANDOM_COUNT',
   SCRIPTING_UTTEXPANSION_INCOMPREHENSION: 'SCRIPTING_UTTEXPANSION_INCOMPREHENSION',
   SCRIPTING_UTTEXPANSION_USENAMEASINTENT: 'SCRIPTING_UTTEXPANSION_USENAMEASINTENT',
+  // justLineTag, utterance
+  SCRIPTING_UTTEXPANSION_NAMING_MODE: 'SCRIPTING_UTTEXPANSION_NAMING_MODE',
+  SCRIPTING_UTTEXPANSION_NAMING_UTTERANCE_MAX: 'SCRIPTING_UTTEXPANSION_NAMING_UTTERANCE_MAX',
   // Del original convo or not
   SCRIPTING_MEMORYEXPANSION_KEEP_ORIG: 'SCRIPTING_MEMORYEXPANSION_KEEP_ORIG',
   // word, non_whitespace, joker
@@ -147,6 +150,9 @@ module.exports = {
   LOGIC_HOOKS: 'LOGIC_HOOKS',
   USER_INPUTS: 'USER_INPUTS',
   // API Calls Rate Limiting
-  RATELIMIT_USERSAYS_MAXCONCURRENT: 'RATELIMIT_RATELIMIT_USERSAYS_MAXCONCURRENTMAXCONCURRENT',
-  RATELIMIT_USERSAYS_MINTIME: 'RATELIMIT_USERSAYS_MINTIME'
+  RATELIMIT_USERSAYS_MAXCONCURRENT: 'RATELIMIT_USERSAYS_MAXCONCURRENT',
+  RATELIMIT_USERSAYS_MINTIME: 'RATELIMIT_USERSAYS_MINTIME',
+  SECURITY_ALLOW_UNSAFE: 'SECURITY_ALLOW_UNSAFE',
+  PRECOMPILERS: 'PRECOMPILERS'
+
 }

src/Defaults.js

@@ -51,10 +51,13 @@ module.exports = {
     [Capabilities.SCRIPTING_MATCHING_MODE]: 'wildcardIgnoreCase',
     [Capabilities.SCRIPTING_UTTEXPANSION_MODE]: 'all',
     [Capabilities.SCRIPTING_UTTEXPANSION_RANDOM_COUNT]: 1,
+    [Capabilities.SCRIPTING_UTTEXPANSION_NAMING_MODE]: 'justLineTag',
+    [Capabilities.SCRIPTING_UTTEXPANSION_NAMING_UTTERANCE_MAX]: '16',
     [Capabilities.SCRIPTING_MEMORYEXPANSION_KEEP_ORIG]: false,
     [Capabilities.ASSERTERS]: [],
     [Capabilities.LOGIC_HOOKS]: [],
-    [Capabilities.USER_INPUTS]: []
+    [Capabilities.USER_INPUTS]: [],
+    [Capabilities.SECURITY_ALLOW_UNSAFE]: true
   },
   Sources: {
     [Source.LOCALPATH]: '.',

src/Plugins.js

@@ -127,6 +127,7 @@ const getPlugins = async (type, resourcesDir) => {
   let items
   try {
     items = fs.readdirSync(pathToRes)
+      .filter(item => path.extname(item) === '.js' || item.indexOf('.') === -1)
   } catch (err) {
     debug(`Cant load plugins, failed to read directory ${pathToRes} - ${err.message}`)
     return []
@@ -153,5 +154,6 @@ module.exports = {
   PLUGIN_TYPE_CONNECTOR,
   PLUGIN_TYPE_ASSERTER,
   PLUGIN_TYPE_LOGICHOOK,
-  PLUGIN_TYPE_USERINPUT
+  PLUGIN_TYPE_USERINPUT,
+  TYPE_TO_PREFIX
 }

src/containers/BaseContainer.js

@@ -4,12 +4,14 @@ const rimraf = require('rimraf')
 const Bottleneck = require('bottleneck')
 const _ = require('lodash')
 const debug = require('debug')('botium-connector-BaseContainer')
+const path = require('path')
 
 const Events = require('../Events')
 const Capabilities = require('../Capabilities')
 const Queue = require('../helpers/Queue')
 const { executeHook, getHook } = require('../helpers/HookUtils')
 const BotiumMockMessage = require('../mocks/BotiumMockMessage')
+const { BotiumError } = require('../scripting/BotiumError')
 
 module.exports = class BaseContainer {
   constructor (eventEmitter, tempDirectory, repo, caps, envs) {
@@ -30,6 +32,34 @@ module.exports = class BaseContainer {
     this.onBotResponseHook = getHook(this.caps[Capabilities.CUSTOMHOOK_ONBOTRESPONSE])
     this.onStopHook = getHook(this.caps[Capabilities.CUSTOMHOOK_ONSTOP])
     this.onCleanHook = getHook(this.caps[Capabilities.CUSTOMHOOK_ONCLEAN])
+
+    if (!this.caps[Capabilities.SECURITY_ALLOW_UNSAFE] &&
+      (
+        this.caps[Capabilities.CUSTOMHOOK_ONBUILD] ||
+        this.caps[Capabilities.CUSTOMHOOK_ONSTART] ||
+        this.caps[Capabilities.CUSTOMHOOK_ONUSERSAYS] ||
+        this.caps[Capabilities.CUSTOMHOOK_ONBOTRESPONSE] ||
+        this.caps[Capabilities.CUSTOMHOOK_ONSTOP] ||
+        this.caps[Capabilities.CUSTOMHOOK_ONCLEAN])) {
+      throw new BotiumError(
+        'Security Error. Using unsafe custom hooks is not allowed',
+        {
+          type: 'security',
+          subtype: 'allow unsafe',
+          source: path.basename(__filename),
+          cause: {
+            SECURITY_ALLOW_UNSAFE: this.caps[Capabilities.SECURITY_ALLOW_UNSAFE],
+            onBuildHook: !!this.caps[Capabilities.CUSTOMHOOK_ONBUILD],
+            onStartHook: !!this.caps[Capabilities.CUSTOMHOOK_ONSTART],
+            onUserSaysHook: !!this.caps[Capabilities.CUSTOMHOOK_ONUSERSAYS],
+            onBotResponseHook: !!this.caps[Capabilities.CUSTOMHOOK_ONBOTRESPONSE],
+            onStopHook: !!this.caps[Capabilities.CUSTOMHOOK_ONSTOP],
+            onCleanHook: !!this.caps[Capabilities.CUSTOMHOOK_ONCLEAN]
+          }
+        }
+      )
+    }
+
     return Promise.resolve()
   }
 

src/containers/plugins/SimpleRestContainer.js

@@ -13,7 +13,7 @@ const { startProxy } = require('../../grid/inbound/proxy')
 const botiumUtils = require('../../helpers/Utils')
 const { getAllCapValues } = require('../../helpers/CapabilitiesUtils')
 const Capabilities = require('../../Capabilities')
-const Defaults = require('../../Defaults')
+const Defaults = require('../../Defaults').Capabilities
 const { SCRIPTING_FUNCTIONS } = require('../../scripting/ScriptingMemory')
 const { getHook, executeHook } = require('../../helpers/HookUtils')
 const { escapeJSONString } = require('../../helpers/Utils')
@@ -25,7 +25,7 @@ const REDIS_TOPIC = 'SIMPLEREST_INBOUND_SUBSCRIPTION'
 module.exports = class SimpleRestContainer {
   constructor ({ queueBotSays, caps }) {
     this.queueBotSays = queueBotSays
-    this.caps = caps
+    this.caps = Object.assign({}, Defaults, caps)
     this.processInbound = false
   }
 
@@ -37,6 +37,7 @@ module.exports = class SimpleRestContainer {
       _.isObject(this.caps[Capabilities.SIMPLEREST_INIT_CONTEXT]) || JSON.parse(this.caps[Capabilities.SIMPLEREST_INIT_CONTEXT])
     }
     if (this.caps[Capabilities.SIMPLEREST_CONTEXT_MERGE_OR_REPLACE] !== 'MERGE' && this.caps[Capabilities.SIMPLEREST_CONTEXT_MERGE_OR_REPLACE] !== 'REPLACE') throw new Error('SIMPLEREST_CONTEXT_MERGE_OR_REPLACE capability only MERGE or REPLACE allowed')
+
     this.startHook = getHook(this.caps[Capabilities.SIMPLEREST_START_HOOK])
     this.stopHook = getHook(this.caps[Capabilities.SIMPLEREST_STOP_HOOK])
     this.requestHook = getHook(this.caps[Capabilities.SIMPLEREST_REQUEST_HOOK])
@@ -65,8 +66,9 @@ module.exports = class SimpleRestContainer {
             // (render(text) is required for forcing mustache to replace valiables in the text first,
             // then send it to the function.)
             // (mapKeys: remove starting $)
-            fnc: _.mapValues(_.mapKeys(SCRIPTING_FUNCTIONS, (value, key) => key.substring(1)), (theFunction) => {
-              return theFunction.length ? function () { return (text, render) => theFunction(render(text)) } : theFunction
+            fnc: _.mapValues(_.mapKeys(SCRIPTING_FUNCTIONS, (value, key) => key.substring(1)), (descriptor) => {
+              const safeCaps = Object.assign({}, this.caps, { [Capabilities.SECURITY_ALLOW_UNSAFE]: true })
+              return descriptor.numberOfArguments ? () => { return (text, render) => descriptor.handler(safeCaps, render(text)) } : () => descriptor.handler(safeCaps)
             })
           }
           this.view.fnc.jsonify = () => (val, render) => {
@@ -363,7 +365,7 @@ module.exports = class SimpleRestContainer {
     }
 
     const uri = this._getMustachedCap(Capabilities.SIMPLEREST_URL, false)
-    const timeout = this.caps[Capabilities.SIMPLEREST_TIMEOUT] || Defaults[Capabilities.SIMPLEREST_TIMEOUT]
+    const timeout = this.caps[Capabilities.SIMPLEREST_TIMEOUT]
 
     const requestOptions = {
       uri,
@@ -453,14 +455,15 @@ module.exports = class SimpleRestContainer {
   }
 
   _getMustachedVal (template, json) {
+    const raw = Mustache.render(template, this.view)
     if (json) {
       try {
-        return JSON.parse(Mustache.render(template, this.view))
+        return JSON.parse(raw)
       } catch (err) {
         return new Error(`JSON parsing failed - try to use {{#fnc.jsonify}}{{xxx}}{{/fnc.jsonify}} to escape JSON special characters (ERR: ${err.message})`)
       }
     } else {
-      return Mustache.render(template, this.view)
+      return raw
     }
   }
 
@@ -569,7 +572,7 @@ module.exports = class SimpleRestContainer {
     if (this.caps[Capabilities.SIMPLEREST_POLL_URL]) {
       const uri = this._getMustachedCap(Capabilities.SIMPLEREST_POLL_URL, false)
       const verb = this.caps[Capabilities.SIMPLEREST_POLL_VERB]
-      const timeout = this.caps[Capabilities.SIMPLEREST_POLL_TIMEOUT] || Defaults[Capabilities.SIMPLEREST_POLL_TIMEOUT]
+      const timeout = this.caps[Capabilities.SIMPLEREST_POLL_TIMEOUT]
       const pollConfig = {
         method: verb,
         uri: uri,
@@ -641,7 +644,7 @@ module.exports = class SimpleRestContainer {
   async _makeCall (capPrefix) {
     const uri = this._getMustachedCap(`${capPrefix}_URL`, false)
     const verb = this.caps[`${capPrefix}_VERB`]
-    const timeout = this.caps[`${capPrefix}_TIMEOUT`] || Defaults[`${capPrefix}_TIMEOUT`] || Defaults[Capabilities.SIMPLEREST_TIMEOUT]
+    const timeout = this.caps[`${capPrefix}_TIMEOUT`] || this.caps[Capabilities.SIMPLEREST_TIMEOUT]
     const httpConfig = {
       method: verb,
       uri: uri,
@@ -665,7 +668,7 @@ module.exports = class SimpleRestContainer {
     }
     this._addRequestOptions(httpConfig)
 
-    const retries = this.caps[`${capPrefix}_RETRIES`] || Defaults[`${capPrefix}_RETRIES`]
+    const retries = this.caps[`${capPrefix}_RETRIES`]
     const response = await this._waitForUrlResponse(httpConfig, retries)
     return response
   }

src/containers/plugins/index.js

@@ -5,6 +5,8 @@ const _ = require('lodash')
 const debug = require('debug')('botium-connector-PluginConnectorContainer-helper')
 
 const SimpleRestContainer = require('./SimpleRestContainer')
+const Capabilities = require('../../Capabilities')
+const { BotiumError } = require('../../scripting/BotiumError')
 
 const pluginResolver = (containermode) => {
   if (containermode === 'simplerest') {
@@ -53,13 +55,31 @@ const loadConnectorModule = (PluginClass, args) => {
 
 const tryLoadPlugin = (containermode, modulepath, args) => {
   const pluginLoaderSpec = modulepath || containermode
+  const _checkUnsafe = (caps, mode, cause) => {
+    if (!caps[Capabilities.SECURITY_ALLOW_UNSAFE]) {
+      throw new BotiumError(
+        `Security Error. Using unsafe connector mode "${mode}" is not allowed`,
+        {
+          type: 'security',
+          subtype: 'allow unsafe',
+          source: 'src/containers/plugins/index.js',
+          cause: {
+            SECURITY_ALLOW_UNSAFE: caps[Capabilities.SECURITY_ALLOW_UNSAFE],
+            mode: mode,
+            ...cause
+          }
+        }
+      )
+    }
+  }
 
   if (pluginResolver(pluginLoaderSpec)) {
     const pluginInstance = new (pluginResolver(pluginLoaderSpec))(args)
     debug('Botium plugin loaded from internal plugin resolver')
     return pluginInstance
   }
   if (_.isFunction(pluginLoaderSpec)) {
+    _checkUnsafe(args.caps, 'Function call', { modulepath, containermode })
     const pluginInstance = pluginLoaderSpec(args)
     debug('Botium plugin loaded from function call')
     return pluginInstance
@@ -69,6 +89,7 @@ const tryLoadPlugin = (containermode, modulepath, args) => {
   if (_.isString(pluginLoaderSpec)) {
     const tryLoadFile = path.resolve(process.cwd(), pluginLoaderSpec)
     if (fs.existsSync(tryLoadFile)) {
+      _checkUnsafe(args.caps, 'Using work dir', { modulepath, containermode })
       try {
         const plugin = require(tryLoadFile)
         if (!plugin.PluginVersion || !plugin.PluginClass) {
@@ -82,33 +103,32 @@ const tryLoadPlugin = (containermode, modulepath, args) => {
         loadErr.push(`Loading Botium plugin from ${tryLoadFile} failed - ${err.message}`)
       }
     }
-    if (pluginLoaderSpec.startsWith('botium-connector-')) {
-      try {
-        const plugin = require(pluginLoaderSpec)
-        if (!plugin.PluginVersion || !plugin.PluginClass) {
-          loadErr.push(`Invalid Botium plugin loaded from ${pluginLoaderSpec}, expected PluginVersion, PluginClass fields`)
-        } else {
-          const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
-          debug(`Botium plugin loaded from ${pluginLoaderSpec}. Plugin version is ${getModuleVersionSafe(pluginLoaderSpec)}`)
-          return pluginInstance
-        }
-      } catch (err) {
-        loadErr.push(`Loading Botium plugin from ${pluginLoaderSpec} failed - ${err.message}`)
+
+    try {
+      const plugin = require(pluginLoaderSpec)
+      if (!plugin.PluginVersion || !plugin.PluginClass) {
+        loadErr.push(`Invalid Botium plugin loaded from ${pluginLoaderSpec}, expected PluginVersion, PluginClass fields`)
+      } else {
+        const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
+        debug(`Botium plugin loaded from ${pluginLoaderSpec}. Plugin version is ${getModuleVersionSafe(pluginLoaderSpec)}`)
+        return pluginInstance
       }
-    } else {
-      const tryLoadPackage = `botium-connector-${pluginLoaderSpec}`
-      try {
-        const plugin = require(tryLoadPackage)
-        if (!plugin.PluginVersion || !plugin.PluginClass) {
-          loadErr.push(`Invalid Botium plugin ${tryLoadPackage}, expected PluginVersion, PluginClass fields`)
-        } else {
-          const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
-          debug(`Botium plugin ${tryLoadPackage} loaded. Plugin version is ${getModuleVersionSafe(tryLoadPackage)}`)
-          return pluginInstance
-        }
-      } catch (err) {
-        loadErr.push(`Loading Botium plugin ${tryLoadPackage} failed, try "npm install ${tryLoadPackage}" - ${err.message}`)
+    } catch (err) {
+      loadErr.push(`Loading Botium plugin from ${pluginLoaderSpec} failed - ${err.message}`)
+    }
+
+    const tryLoadPackage = `botium-connector-${pluginLoaderSpec}`
+    try {
+      const plugin = require(tryLoadPackage)
+      if (!plugin.PluginVersion || !plugin.PluginClass) {
+        loadErr.push(`Invalid Botium plugin ${tryLoadPackage}, expected PluginVersion, PluginClass fields`)
+      } else {
+        const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
+        debug(`Botium plugin ${tryLoadPackage} loaded. Plugin version is ${getModuleVersionSafe(tryLoadPackage)}`)
+        return pluginInstance
       }
+    } catch (err) {
+      loadErr.push(`Loading Botium plugin ${tryLoadPackage} failed, try "npm install ${tryLoadPackage}" - ${err.message}`)
     }
   }
   throw new Error(`Loading Botium Plugin failed.\r\n${loadErr.join('\r\n')}`)

src/helpers/Utils.js

@@ -8,8 +8,17 @@ const isStringJson = (string) => {
   }
   return true
 }
-
-const isJsonObject = (json) => {
+/**
+ *
+ * @param json
+ * @param stringIsJson It is possible to stringify a string, so we can say its a json object,
+ * but usually thats not what we expect. Default is true because backward compatibility
+ * @returns {boolean}
+ */
+const isJsonObject = (json, stringIsJson = true) => {
+  if (!stringIsJson && _.isString(json)) {
+    return false
+  }
   try {
     JSON.stringify(json)
   } catch (e) {