diff --git a/.gitignore b/.gitignore index 3c3629e64..eb03e3e1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +*.log diff --git a/import.js b/import.js index c366414c9..e86cbf2ee 100644 --- a/import.js +++ b/import.js @@ -11,10 +11,9 @@ var logger = require( 'pelias-logger' ).get( 'openaddresses' ); var parameters = require( './lib/parameters' ); var importPipeline = require( './lib/importPipeline' ); -var adminLookupStream = require('./lib/streams/adminLookupStream'); +const adminLookupStream = require('pelias-wof-admin-lookup'); var deduplicatorStream = require('./lib/streams/deduplicatorStream'); -var wofAdminLookup = require('pelias-wof-admin-lookup'); var addressDeduplicator = require('pelias-address-deduplicator'); @@ -37,10 +36,14 @@ if( 'exitCode' in args ){ } else { startTiming(); + if (peliasConfig.imports.openaddresses.hasOwnProperty('adminLookup')) { + logger.info('imports.openaddresses.adminLookup has been deprecated, ' + + 'enabled adminLookup using imports.adminLookup.enabled = true'); + } + var files = parameters.getFileList(peliasConfig, args); var deduplicator = deduplicatorStream.create(peliasConfig, addressDeduplicator); - var adminLookup = adminLookupStream.create(peliasConfig, wofAdminLookup); - importPipeline.create( files, args.dirPath, deduplicator, adminLookup ); + importPipeline.create( files, args.dirPath, deduplicator, adminLookupStream.create() ); } diff --git a/lib/streams/adminLookupStream.js b/lib/streams/adminLookupStream.js deleted file mode 100644 index 850d363ff..000000000 --- a/lib/streams/adminLookupStream.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict'; - -const through = require( 'through2' ); -const _ = require('lodash'); -const logger = require( 'pelias-logger' ).get( 'openaddresses' ); - -// helper function that verifies that admin lookup should run -function adminLookupIsEnabled(config) { - return _.get(config, 'imports.openaddresses.adminLookup', false); -} - -function createAdminLookupStream(config,adminLookup) { - if (adminLookupIsEnabled(config)) { - logger.info( 'Setting up admin value lookup stream.' ); - const pipResolver = adminLookup.createLocalWofPipResolver(); - return adminLookup.createLookupStream(pipResolver); - - } else { - return through.obj(function (doc, enc, next) { - next(null, doc); - }); - - } - -} - -module.exports = { - create: createAdminLookupStream -}; diff --git a/package.json b/package.json index 9010c93c5..0df828d38 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "pelias-dbclient": "2.0.0", "pelias-logger": "0.1.0", "pelias-model": "4.4.0", - "pelias-wof-admin-lookup": "2.11.0", + "pelias-wof-admin-lookup": "3.0.0", "through2": "^2.0.3", "through2-filter": "^2.0.0", "through2-map": "^3.0.0", diff --git a/test/streams/adminLookupStream.js b/test/streams/adminLookupStream.js deleted file mode 100644 index 89839947c..000000000 --- a/test/streams/adminLookupStream.js +++ /dev/null @@ -1,158 +0,0 @@ -var sink = require('through2-sink'); -var adminLookup = require('../../lib/streams/adminLookupStream').create; -var tape = require('tape'); - - -tape('enabled: create pipResolver', function (t) { - var config = { - imports: { - openaddresses:{ - adminLookup: true - }, - adminlookup: { - url: 'anything.org' - } - } - }; - -var wofAdminLookup = { - createLocalWofPipResolver: function(){}, - createLookupStream: function(){} - }; - // assert that the PipResolver was instantiated with the correct URL - wofAdminLookup.createLocalWofPipResolver = function () { - t.pass('Resolver created'); - t.end(); - }; - - adminLookup(config, wofAdminLookup); - }); - -tape('enabled: pip resolver is passed into stream constructor', function (t) { - var config = { - imports: { - openaddresses:{ - adminLookup: true - }, - adminlookup: { - url: 'anything.org' - } - } - }; - -var wofAdminLookup = { - createLocalWofPipResolver: function(){}, - createLookupStream: function(){} - }; - - t.plan(1); // expect 3 assertions - - var pipResolverMock = {foo: 'bar'}; - - // mock the creation of pip resolver - wofAdminLookup.createLocalWofPipResolver = function () { - return pipResolverMock; - }; - - wofAdminLookup.createLookupStream = function (pipResolver) { - t.equal(pipResolver, pipResolverMock); - t.end(); - }; - - adminLookup(config, wofAdminLookup); - }); - - /* - * There was a bug (https://github.com/pelias/wof-admin-lookup/issues/51) where admin lookup could - * not be enabled without the adminLookup config section - */ - tape('enabled without any special adminLookup config: return pip stream', function (t) { - var config = { - imports: { - openaddresses: { - adminLookup: true - } - } - }; - - t.plan(1); - - var streamMock = {madeBy: 'mock'}; - - var wofAdminLookup = { - createLocalWofPipResolver: function() { - }, - createLookupStream: function() { - return streamMock; - } - }; - - var stream = adminLookup(config, wofAdminLookup); - t.equal(stream, streamMock, 'stream created'); - t.end(); - }); - - tape('disabled: return passthrough stream', function(t) { - var config = { - imports: { - openaddresses: { - adminLookup: false - } - } - }; - - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); - - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); - - stream.write(dataItem); - }); - - tape('absence of config.imports should return pass-through stream', function(t) { - var config = {}; - - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); - - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); - - stream.write(dataItem); - }); - - tape('absence of config.imports.openaddresses should return pass-through stream', function(t) { - var config = { - imports: {} - }; - - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); - - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); - - stream.write(dataItem); - }); diff --git a/test/test.js b/test/test.js index f572ff3c0..bfa49b46d 100644 --- a/test/test.js +++ b/test/test.js @@ -6,7 +6,6 @@ require( './schema' ); require( './isValidCsvRecord' ); -require( './streams/adminLookupStream'); require( './import'); require( './importPipeline'); require( './parameters' );