Global configuration option to select stored languages

Document.js

@@ -1,4 +1,4 @@
-const config = require('pelias-config').generate();
+const config = require('pelias-config').generate(require('./schema'));
 const validate = require('./util/valid');
 const transform = require('./util/transform');
 const _ = require('lodash');
@@ -44,6 +44,8 @@ function Document( source, layer, source_id ){
   this.addPostProcessingScript( require('./post/intersections') );
   this.addPostProcessingScript( require('./post/seperable_street_names').post );
   this.addPostProcessingScript( require('./post/deduplication') );
+  this.addPostProcessingScript( require('./post/language_default')(config) );
+  this.addPostProcessingScript( require('./post/language_field_filter')(config) );
   this.addPostProcessingScript( require('./post/language_field_trimming') );
 
   // mandatory properties

package.json

@@ -28,6 +28,7 @@
     "node": ">= 10.0.0"
   },
   "dependencies": {
+    "joi": "^17.1.1",
     "lodash": "^4.6.1",
     "pelias-config": "^4.5.0",
     "through2": "^3.0.0"

post/language_default.js

@@ -0,0 +1,19 @@
+const _ = require('lodash');
+const prefixes = ['name', 'phrase'];
+
+module.exports = config => {
+  return function defaultLang(doc) {
+    const defaultLang = config.get('imports.langs.default');
+    if (!_.isString(defaultLang)) { return; }
+
+    prefixes.forEach((prefix) => {
+      let field = doc[prefix];
+
+      if (!_.isPlainObject(field)) { return; }
+
+      if (!_.isEmpty(field[defaultLang])) {
+        field['default'] = _.concat(field[defaultLang], field['default']).filter((e) => _.isString(e));
+      }
+    });
+  };
+};

post/language_field_filter.js

@@ -0,0 +1,33 @@
+const _ = require('lodash');
+const prefixes = ['name', 'phrase'];
+
+function normalizeConfig(config) {
+  const keep = config.get('imports.langs.keep') || config.get('imports.langs');
+  if (_.isString(keep)) {
+    return new Set([keep]);
+  }
+  if (_.isArray(keep)) {
+    return new Set(keep);
+  }
+}
+
+module.exports = (config) => {
+  const keep = normalizeConfig(config);
+  return function filter (doc) {
+    if (!keep) { return; }
+
+    prefixes.forEach((prefix) => {
+      let field = doc[prefix];
+
+      if (!_.isPlainObject(field)) {
+        return;
+      }
+
+      _.each(field, (_names, lang) => {
+        if (lang !== 'default' && !keep.has(lang)) {
+          delete field[lang];
+        }
+      });
+    });
+  };
+};

schema.js

@@ -0,0 +1,18 @@
+
+const Joi = require('joi');
+
+module.exports = Joi.object().keys({
+  imports: Joi.object().keys({
+    langs: Joi.alternatives().try(
+      Joi.string().regex(/^[a-z]{2}$/),
+      Joi.array().items(Joi.string().regex(/^[a-z]{2}$/)).min(1),
+      Joi.object().keys({
+        keep: Joi.alternatives().try(
+          Joi.string().regex(/^[a-z]{2}$/),
+          Joi.array().items(Joi.string().regex(/^[a-z]{2}$/)).min(1)
+        ),
+        default: Joi.string().regex(/^[a-z]{2}$/)
+      })
+    )
+  }).unknown(true)
+}).unknown(true);

test/document/post.js

@@ -1,17 +1,26 @@
 
-const Document = require('../../Document');
+const _ = require('lodash');
+const proxyquire = require('proxyquire');
+const config = { get: _.get.bind(null, {}) };
+const Document = proxyquire('../../Document', { 'pelias-config': config });
 const intersections = require('../../post/intersections');
 const seperable_street_names = require('../../post/seperable_street_names').post;
 const deduplication = require('../../post/deduplication');
 const language_field_trimming = require('../../post/language_field_trimming');
-const DEFAULT_SCRIPTS = [intersections, seperable_street_names, deduplication, language_field_trimming ];
+const language_field_filter = require('../../post/language_field_filter');
+const language_default = require('../../post/language_default');
+const DEFAULT_SCRIPT_NAMES = [
+  intersections, seperable_street_names, deduplication,
+  language_default(config), language_field_filter(config), language_field_trimming
+].map(f => f.name);
 
 module.exports.tests = {};
 
 module.exports.tests.addPostProcessingScript = function(test) {
   test('default scripts', function(t) {
     let doc = new Document('mysource','mylayer','myid');
-    t.deepEqual(doc._post, DEFAULT_SCRIPTS, 'default processing scripts');
+    t.equal(doc._post.length, 6);
+    t.deepEqual(doc._post.map(f => f.name), DEFAULT_SCRIPT_NAMES, 'default processing scripts');
     t.end();
   });
   test('invalid type', function(t) {
@@ -26,15 +35,19 @@ module.exports.tests.addPostProcessingScript = function(test) {
     let script = function(){};
     let doc = new Document('mysource','mylayer','myid');
     doc.addPostProcessingScript( script );
-    t.deepEqual(doc._post, DEFAULT_SCRIPTS.concat( script ), 'default processing scripts');
+    t.deepEqual(doc._post.map(f => f.name), DEFAULT_SCRIPT_NAMES.concat( script.name ), 'default processing scripts');
     t.end();
   });
   test('set same function twice (allowed)', function(t) {
     let script = function(){};
     let doc = new Document('mysource','mylayer','myid');
     doc.addPostProcessingScript( script );
     doc.addPostProcessingScript( script );
-    t.deepEqual(doc._post, DEFAULT_SCRIPTS.concat( script, script ), 'default processing scripts');
+    t.deepEqual(
+      doc._post.map(f => f.name),
+      DEFAULT_SCRIPT_NAMES.concat( script.name, script.name ),
+      'default processing scripts'
+    );
     t.end();
   });
 };

test/document/toESDocument.js

@@ -1,5 +1,6 @@
 const proxyquire = require('proxyquire');
 const codec = require('../../codec');
+const _ = require('lodash');
 
 var fakeGeneratedConfig = {
   schema: {
@@ -10,7 +11,7 @@ var fakeGeneratedConfig = {
 
 const fakeConfig = {
   generate: function fakeGenerate() {
-    return fakeGeneratedConfig;
+    return Object.assign({ get: _.get.bind(null, fakeGeneratedConfig) }, fakeGeneratedConfig);
   }
 };
 

test/post/language_default.js

@@ -0,0 +1,115 @@
+const language_default = require('../../post/language_default');
+const Document = require('../../Document');
+const _ = require('lodash');
+
+const generateFakeConfig = (config) => {
+  return { get: _.get.bind(null, config) };
+};
+
+module.exports.tests = {};
+
+module.exports.tests.default = function (test) {
+  test('default - empty config', function (t) {
+    const fakeConfig = generateFakeConfig({});
+    const doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setNameAlias('default', 'test2');
+    doc.setNameAlias('default', 'test3');
+
+    doc.setName('en', 'test4');
+    doc.setNameAlias('en', 'test3');
+    doc.setNameAlias('en', 'test5');
+
+    language_default(fakeConfig)(doc);
+
+    t.deepEquals(doc.name.default, ['test1', 'test2', 'test3']);
+    t.deepEquals(doc.phrase.default, ['test1', 'test2', 'test3']);
+
+    t.end();
+  });
+
+  test('default - en with aliases', function (t) {
+    const fakeConfig = generateFakeConfig({ imports: { langs: { default: 'en' } } });
+    const doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setNameAlias('default', 'test2');
+    doc.setNameAlias('default', 'test3');
+
+    doc.setName('en', 'test4');
+    doc.setNameAlias('en', 'test3');
+    doc.setNameAlias('en', 'test5');
+
+    language_default(fakeConfig)(doc);
+
+    t.deepEquals(doc.name.default, ['test4', 'test3', 'test5', 'test1', 'test2', 'test3']);
+    t.deepEquals(doc.phrase.default, ['test4', 'test3', 'test5', 'test1', 'test2', 'test3']);
+
+    t.end();
+  });
+
+  test('default - en without aliases', function (t) {
+    const fakeConfig = generateFakeConfig({ imports: { langs: { default: 'en' } } });
+
+    const doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setNameAlias('default', 'test2');
+    doc.setNameAlias('default', 'test3');
+
+    doc.setName('en', 'test4');
+
+    language_default(fakeConfig)(doc);
+
+    t.deepEquals(doc.name.default, ['test4', 'test1', 'test2', 'test3']);
+    t.deepEquals(doc.phrase.default, ['test4', 'test1', 'test2', 'test3']);
+
+    t.end();
+  });
+
+  test('default - without en name', function (t) {
+    const fakeConfig = generateFakeConfig({ imports: { langs: { default: 'en' } } });
+
+    const doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setNameAlias('default', 'test2');
+    doc.setNameAlias('default', 'test3');
+
+    language_default(fakeConfig)(doc);
+
+    t.deepEquals(doc.name.default, ['test1', 'test2', 'test3']);
+    t.deepEquals(doc.phrase.default, ['test1', 'test2', 'test3']);
+
+    t.end();
+  });
+
+  test('default - without default name', function (t) {
+    const fakeConfig = generateFakeConfig({ imports: { langs: { default: 'en' } } });
+
+    const doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('en', 'test1');
+
+    language_default(fakeConfig)(doc);
+
+    t.deepEquals(doc.name.default, ['test1']);
+    t.deepEquals(doc.phrase.default, ['test1']);
+
+    t.deepEquals(doc.name.en, 'test1');
+    t.deepEquals(doc.phrase.en, 'test1');
+
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('post/language_default: ' + name, testFunction);
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test, common);
+  }
+};