From 05943204e266c739814d4f631a910e9284826030 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sat, 19 May 2018 22:59:54 -0400 Subject: [PATCH] feat(libpostal): Use libpostal service BREAKING CHANGE Use microservice-wrapper to avoid having to load libpostal locally. Note: this now requires a new configuration section in `pelias.json`, a top-level `services` key with the usual properties. Here's an example full `pelias.json`: ``` { "api": { "textAnalyzer": "libpostal" }, "services": { "libpostal": { "url": "http://libpostal-service-url:8080", "timeout": 4000 } } } ``` Fixes https://github.com/pelias/interpolation/issues/106 --- lib/analyze.js | 24 ++----------------- lib/libpostal_wrapper.js | 22 +++++++++++++++++ libpostal/service.js | 48 ++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ test/lib/mock_libpostal.js | 2 +- 5 files changed, 75 insertions(+), 23 deletions(-) create mode 100644 lib/libpostal_wrapper.js create mode 100644 libpostal/service.js diff --git a/lib/analyze.js b/lib/analyze.js index cc13cf99..381ed599 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -1,3 +1,4 @@ +const libpostal_service = require( './libpostal_wrapper' ); // constants for controlling how we parse ranges, eg: 'α-β' // some ranges such as '1-7' are ambiguous; it could mean 'apt 7, no 1'; or // it could mean 'apt 1, no 7'; or could even be a valid range 'one to seven'. @@ -7,32 +8,11 @@ var MIN_RANGE = 1; // the miniumum amount β is higher than α var MAX_RANGE = 6; // the maximum amount β is higher than α var MIN_RANGE_HOUSENUMBER = 10; // the minimum acceptible value for both α and β -/* - * Return the appropriate version of node-postal - */ - -var _nodepostal_module; -function get_libpostal() { - // lazy load this dependency; since it's large (~2GB RAM) and may be - // accidentally required by a process which doesn't use it. - if (!_nodepostal_module) { - // load the mock library if MOCK_LIBPOSTAL env var is set - if (process.env.MOCK_LIBPOSTAL) { - _nodepostal_module = require('../test/lib/mock_libpostal'); - // otherwise load the real thing - } else { - _nodepostal_module = require('node-postal'); - } - } - - return _nodepostal_module; -} - /** analyze input streetname string and return a list of expansions. **/ function street( streetName ){ - const postal = get_libpostal(); + const postal = libpostal_service(); // use libpostal to expand the address var expansions = postal.expand.expand_address( streetName ); diff --git a/lib/libpostal_wrapper.js b/lib/libpostal_wrapper.js new file mode 100644 index 00000000..fe6048a1 --- /dev/null +++ b/lib/libpostal_wrapper.js @@ -0,0 +1,22 @@ +const mock_libpostal = require('../test/lib/mock_libpostal'); + +// This module is a wrapper around the actual libpostal service library +// and the mock libpostal library +// it allows an environment variable to switch which library is used in application code + +let libpostal_module; +function get_libpostal() { + // return the mock library if MOCK_LIBPOSTAL env var is set + if (process.env.MOCK_LIBPOSTAL) { + return mock_libpostal; + // otherwise return the actual service + } else { + // lazy load the libpostal module so that tests can skip configuring the service + if (!libpostal_module) { + libpostal_module = require( '../libpostal/service' ); + } + return libpostal_module; + } +} + +module.exports = get_libpostal; diff --git a/libpostal/service.js b/libpostal/service.js new file mode 100644 index 00000000..aaa8d719 --- /dev/null +++ b/libpostal/service.js @@ -0,0 +1,48 @@ +// deasync is used to proved a sync-looking interface +// to the async call to the libpostal service +const deasync = require('deasync'); +const microservice_wrapper = require('pelias-microservice-wrapper'); +const pelias_config = require('pelias-config').generate(); + +if (!pelias_config.services || !pelias_config.services.libpostal || !pelias_config.services.libpostal.url) { + throw new Error('The URL to the libpostal service must be set in the services.libpostal.url property of pelias.json'); +} + +const LibpostalServiceConfig = class extends microservice_wrapper.ServiceConfiguration { + constructor(configBlob) { + super('libpostal', configBlob); + } + getUrl(params) { + return this.baseUrl + params.endpoint; + } + getParameters(params) { + return { + address: params.address + }; + } +}; + +// create an instance of the libpostal service, with a synchronous interface +const libpostal_service = deasync(microservice_wrapper.service( + new LibpostalServiceConfig(pelias_config.services.libpostal) +)); + +// create an object that looks like the interface to `node-postal` but uses a remote service +module.exports = { + expand: { + expand_address: function(param) { + const params = { + endpoint: 'expand', + address: param + }; + + // the libpostal service will not handle an empty parameter + // so return empty array immediately + if (!param) { + return []; + } + const result = libpostal_service(params); + return result; + } + } +}; diff --git a/package.json b/package.json index bdb345c8..c78ee9c2 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,9 @@ "@mapbox/polyline": "^1.0.0", "async": "^3.1.0", "cli-table3": "^0.5.0", + "csv-parse": "^4.0.0", "csv-parse": "^4.4.6", + "deasync": "^0.1.15", "express": "^4.14.0", "fs-extra": "^8.0.0", "jsftp": "^2.0.0", diff --git a/test/lib/mock_libpostal.js b/test/lib/mock_libpostal.js index 9d064750..d6543b59 100644 --- a/test/lib/mock_libpostal.js +++ b/test/lib/mock_libpostal.js @@ -22,7 +22,7 @@ module.exports.expand = { // is enabled, lazy load real libpostal, and return the real response } else if (use_real_libpostal) { // lazy load libpostal only when needed - if (!real_libpostal) { real_libpostal = require('node-postal'); } + if (!real_libpostal) { real_libpostal = require('../../libpostal/service'); } const real_response = real_libpostal.expand.expand_address(clean_string); mock_responses[clean_string] = real_response;