From c91e9d3df007aecd4b878f6ad2cc2179cd7ff140 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 7 Feb 2017 14:13:24 -0500 Subject: [PATCH 1/7] made log file ignorable to git --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3c3629e64..eb03e3e1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +*.log From f80f633edaa3449c1a8846d05545d136b3e6b6cb Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 7 Feb 2017 14:13:53 -0500 Subject: [PATCH 2/7] use specific version of wof-admin-lookup --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 800fe5133..160770f44 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "pelias-dbclient": "2.0.0", "pelias-logger": "0.1.0", "pelias-model": "4.4.0", - "pelias-wof-admin-lookup": "2.11.0", + "pelias-wof-admin-lookup": "pelias/wof-admin-lookup#incorporate-pip-service-directly", "through2": "^2.0.3", "through2-filter": "^2.0.0", "through2-map": "^3.0.0", From 4c3976ece2431edb1a88ea8b8c1dc04e01c00b23 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 7 Feb 2017 14:14:26 -0500 Subject: [PATCH 3/7] switched to all-in-one initialization for wof-admin-lookup --- import.js | 3 +- lib/streams/adminLookupStream.js | 6 +- test/streams/adminLookupStream.js | 206 ++++++++++++------------------ 3 files changed, 89 insertions(+), 126 deletions(-) diff --git a/import.js b/import.js index 329ad03b8..d360885a9 100644 --- a/import.js +++ b/import.js @@ -16,7 +16,6 @@ var importPipeline = require( './lib/importPipeline' ); var adminLookupStream = require('./lib/streams/adminLookupStream'); var deduplicatorStream = require('./lib/streams/deduplicatorStream'); -var wofAdminLookup = require('pelias-wof-admin-lookup'); var addressDeduplicator = require('pelias-address-deduplicator'); @@ -42,7 +41,7 @@ if( 'exitCode' in args ){ var files = parameters.getFileList(peliasConfig, args); var deduplicator = deduplicatorStream.create(peliasConfig, addressDeduplicator); - var adminLookup = adminLookupStream.create(peliasConfig, wofAdminLookup); + var adminLookup = adminLookupStream.create(peliasConfig); importPipeline.create( files, args.dirPath, deduplicator, adminLookup ); } diff --git a/lib/streams/adminLookupStream.js b/lib/streams/adminLookupStream.js index 850d363ff..a3c7a355e 100644 --- a/lib/streams/adminLookupStream.js +++ b/lib/streams/adminLookupStream.js @@ -3,17 +3,17 @@ const through = require( 'through2' ); const _ = require('lodash'); const logger = require( 'pelias-logger' ).get( 'openaddresses' ); +const wofAdminLookup = require('pelias-wof-admin-lookup'); // helper function that verifies that admin lookup should run function adminLookupIsEnabled(config) { return _.get(config, 'imports.openaddresses.adminLookup', false); } -function createAdminLookupStream(config,adminLookup) { +function createAdminLookupStream(config) { if (adminLookupIsEnabled(config)) { logger.info( 'Setting up admin value lookup stream.' ); - const pipResolver = adminLookup.createLocalWofPipResolver(); - return adminLookup.createLookupStream(pipResolver); + return wofAdminLookup.createLookupStream(); } else { return through.obj(function (doc, enc, next) { diff --git a/test/streams/adminLookupStream.js b/test/streams/adminLookupStream.js index 89839947c..62478b32a 100644 --- a/test/streams/adminLookupStream.js +++ b/test/streams/adminLookupStream.js @@ -1,99 +1,106 @@ -var sink = require('through2-sink'); -var adminLookup = require('../../lib/streams/adminLookupStream').create; -var tape = require('tape'); - - -tape('enabled: create pipResolver', function (t) { - var config = { +const tape = require('tape'); +const event_stream = require('event-stream'); +const through = require('through2'); +const proxyquire = require('proxyquire').noCallThru(); + +const adminLookupStream = require('../../lib/streams/adminLookupStream').create; + +function test_stream(input, testedStream, callback) { + const input_stream = event_stream.readArray(input); + const destination_stream = event_stream.writeArray(callback); + + input_stream.pipe(testedStream).pipe(destination_stream); +} + +/* + * There was a bug (https://github.com/pelias/wof-admin-lookup/issues/51) where admin lookup could + * not be enabled without the adminLookup config section + */ +tape('enabled without any special adminLookup config: return pip stream', function (t) { + const config = { imports: { - openaddresses:{ + openaddresses: { adminLookup: true - }, - adminlookup: { - url: 'anything.org' } } }; -var wofAdminLookup = { - createLocalWofPipResolver: function(){}, - createLookupStream: function(){} - }; - // assert that the PipResolver was instantiated with the correct URL - wofAdminLookup.createLocalWofPipResolver = function () { - t.pass('Resolver created'); - t.end(); - }; - - adminLookup(config, wofAdminLookup); - }); + const infoMessages = []; -tape('enabled: pip resolver is passed into stream constructor', function (t) { - var config = { - imports: { - openaddresses:{ - adminLookup: true - }, - adminlookup: { - url: 'anything.org' + const stream = proxyquire('../../lib/streams/adminLookupStream', { + 'pelias-wof-admin-lookup': { + createLookupStream: () => { + return through.obj(function (doc, enc, next) { + doc.field2 = 'value 2'; + next(null, doc); + }); + } + }, + 'pelias-logger': { + get: (log_module) => { + t.equal(log_module, 'openaddresses'); + return { + info: (message) => { + infoMessages.push(message); + } + }; } } + }).create(config); + + const input = { + field1: 'value 1' }; -var wofAdminLookup = { - createLocalWofPipResolver: function(){}, - createLookupStream: function(){} + const expected = { + field1: 'value 1', + field2: 'value 2' }; - t.plan(1); // expect 3 assertions + test_stream([input], stream, (err, actual) => { + t.deepEqual(actual, [expected], 'should have changed'); + t.deepEqual(infoMessages, ['Setting up admin value lookup stream.']); + t.end(); + }); - var pipResolverMock = {foo: 'bar'}; +}); - // mock the creation of pip resolver - wofAdminLookup.createLocalWofPipResolver = function () { - return pipResolverMock; - }; +tape('absence of config.imports should return pass-through stream', function(t) { + const config = {}; - wofAdminLookup.createLookupStream = function (pipResolver) { - t.equal(pipResolver, pipResolverMock); - t.end(); - }; + const input = { + some: 'data' + }; + + const stream = adminLookupStream(config); - adminLookup(config, wofAdminLookup); + test_stream([input], stream, (err, actual) => { + t.deepEqual(actual, [input], 'nothing should have changed'); + t.end(); }); - /* - * There was a bug (https://github.com/pelias/wof-admin-lookup/issues/51) where admin lookup could - * not be enabled without the adminLookup config section - */ - tape('enabled without any special adminLookup config: return pip stream', function (t) { - var config = { - imports: { - openaddresses: { - adminLookup: true - } - } +}); + +tape('absence of config.imports.openaddresses should return pass-through stream', function(t) { + const config = { + imports: {} }; - t.plan(1); + const input = { + some: 'data' + }; - var streamMock = {madeBy: 'mock'}; + const stream = adminLookupStream(config); - var wofAdminLookup = { - createLocalWofPipResolver: function() { - }, - createLookupStream: function() { - return streamMock; - } - }; - - var stream = adminLookup(config, wofAdminLookup); - t.equal(stream, streamMock, 'stream created'); + test_stream([input], stream, (err, actual) => { + t.deepEqual(actual, [input], 'nothing should have changed'); t.end(); }); - tape('disabled: return passthrough stream', function(t) { - var config = { +}); + +tape('disabled: return passthrough stream', function(t) { + const config = { imports: { openaddresses: { adminLookup: false @@ -101,58 +108,15 @@ var wofAdminLookup = { } }; - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); - - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); - - stream.write(dataItem); - }); - - tape('absence of config.imports should return pass-through stream', function(t) { - var config = {}; - - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); + const input = { + some: 'data' + }; - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); + const stream = adminLookupStream(config); - stream.write(dataItem); + test_stream([input], stream, (err, actual) => { + t.deepEqual(actual, [input], 'nothing should have changed'); + t.end(); }); - tape('absence of config.imports.openaddresses should return pass-through stream', function(t) { - var config = { - imports: {} - }; - - t.plan(2); // expect 2 assertions - - var dataItem = { some: 'data' }; - - var stream = adminLookup(config, {}); - - t.equal(typeof stream, 'object', 'disabled stream is an object'); - - stream.pipe(sink.obj( function (doc) { - t.deepEqual(doc, dataItem); - t.end(); - })); - - stream.write(dataItem); - }); +}); From a4738db4225457dbb00c11a18c6a8fd169351104 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Thu, 9 Feb 2017 14:57:05 -0500 Subject: [PATCH 4/7] switch to direct call to wof-admin-lookup for create stream added logging message about deprecated import.openaddresses.adminLookup setting --- import.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/import.js b/import.js index d360885a9..b8613a3b9 100644 --- a/import.js +++ b/import.js @@ -13,7 +13,7 @@ var logger = require( 'pelias-logger' ).get( 'openaddresses' ); var parameters = require( './lib/parameters' ); var importPipeline = require( './lib/importPipeline' ); -var adminLookupStream = require('./lib/streams/adminLookupStream'); +const adminLookup = require('pelias-wof-admin-lookup').createLookupStream(); var deduplicatorStream = require('./lib/streams/deduplicatorStream'); var addressDeduplicator = require('pelias-address-deduplicator'); @@ -38,10 +38,14 @@ if( 'exitCode' in args ){ } else { startTiming(); + if (peliasConfig.imports.openaddresses.hasOwnProperty('adminLookup')) { + logger.info('imports.openaddresses.adminLookup has been deprecated, ' + + 'enabled adminLookup using imports.adminLookup.enabled = true'); + } + var files = parameters.getFileList(peliasConfig, args); var deduplicator = deduplicatorStream.create(peliasConfig, addressDeduplicator); - var adminLookup = adminLookupStream.create(peliasConfig); importPipeline.create( files, args.dirPath, deduplicator, adminLookup ); } From cfb8b531e8cabab439077c2dde03799a2d1bdf67 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Thu, 9 Feb 2017 14:59:54 -0500 Subject: [PATCH 5/7] removed unused file --- lib/streams/adminLookupStream.js | 29 ------- test/streams/adminLookupStream.js | 122 ------------------------------ test/test.js | 1 - 3 files changed, 152 deletions(-) delete mode 100644 lib/streams/adminLookupStream.js delete mode 100644 test/streams/adminLookupStream.js diff --git a/lib/streams/adminLookupStream.js b/lib/streams/adminLookupStream.js deleted file mode 100644 index a3c7a355e..000000000 --- a/lib/streams/adminLookupStream.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict'; - -const through = require( 'through2' ); -const _ = require('lodash'); -const logger = require( 'pelias-logger' ).get( 'openaddresses' ); -const wofAdminLookup = require('pelias-wof-admin-lookup'); - -// helper function that verifies that admin lookup should run -function adminLookupIsEnabled(config) { - return _.get(config, 'imports.openaddresses.adminLookup', false); -} - -function createAdminLookupStream(config) { - if (adminLookupIsEnabled(config)) { - logger.info( 'Setting up admin value lookup stream.' ); - return wofAdminLookup.createLookupStream(); - - } else { - return through.obj(function (doc, enc, next) { - next(null, doc); - }); - - } - -} - -module.exports = { - create: createAdminLookupStream -}; diff --git a/test/streams/adminLookupStream.js b/test/streams/adminLookupStream.js deleted file mode 100644 index 62478b32a..000000000 --- a/test/streams/adminLookupStream.js +++ /dev/null @@ -1,122 +0,0 @@ -const tape = require('tape'); -const event_stream = require('event-stream'); -const through = require('through2'); -const proxyquire = require('proxyquire').noCallThru(); - -const adminLookupStream = require('../../lib/streams/adminLookupStream').create; - -function test_stream(input, testedStream, callback) { - const input_stream = event_stream.readArray(input); - const destination_stream = event_stream.writeArray(callback); - - input_stream.pipe(testedStream).pipe(destination_stream); -} - -/* - * There was a bug (https://github.com/pelias/wof-admin-lookup/issues/51) where admin lookup could - * not be enabled without the adminLookup config section - */ -tape('enabled without any special adminLookup config: return pip stream', function (t) { - const config = { - imports: { - openaddresses: { - adminLookup: true - } - } - }; - - const infoMessages = []; - - const stream = proxyquire('../../lib/streams/adminLookupStream', { - 'pelias-wof-admin-lookup': { - createLookupStream: () => { - return through.obj(function (doc, enc, next) { - doc.field2 = 'value 2'; - next(null, doc); - }); - } - }, - 'pelias-logger': { - get: (log_module) => { - t.equal(log_module, 'openaddresses'); - return { - info: (message) => { - infoMessages.push(message); - } - }; - } - } - }).create(config); - - const input = { - field1: 'value 1' - }; - - const expected = { - field1: 'value 1', - field2: 'value 2' - }; - - test_stream([input], stream, (err, actual) => { - t.deepEqual(actual, [expected], 'should have changed'); - t.deepEqual(infoMessages, ['Setting up admin value lookup stream.']); - t.end(); - }); - -}); - -tape('absence of config.imports should return pass-through stream', function(t) { - const config = {}; - - const input = { - some: 'data' - }; - - const stream = adminLookupStream(config); - - test_stream([input], stream, (err, actual) => { - t.deepEqual(actual, [input], 'nothing should have changed'); - t.end(); - }); - -}); - -tape('absence of config.imports.openaddresses should return pass-through stream', function(t) { - const config = { - imports: {} - }; - - const input = { - some: 'data' - }; - - const stream = adminLookupStream(config); - - test_stream([input], stream, (err, actual) => { - t.deepEqual(actual, [input], 'nothing should have changed'); - t.end(); - }); - -}); - -tape('disabled: return passthrough stream', function(t) { - const config = { - imports: { - openaddresses: { - adminLookup: false - } - } - }; - - const input = { - some: 'data' - }; - - const stream = adminLookupStream(config); - - test_stream([input], stream, (err, actual) => { - t.deepEqual(actual, [input], 'nothing should have changed'); - t.end(); - }); - -}); diff --git a/test/test.js b/test/test.js index 72111d640..730c1d060 100644 --- a/test/test.js +++ b/test/test.js @@ -6,7 +6,6 @@ require( './configValidation' ); require( './isValidCsvRecord' ); -require( './streams/adminLookupStream'); require( './import'); require( './importPipeline'); require( './parameters' ); From 00e49c78913cc1e4e09649703fc566af81e5ee00 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Thu, 9 Feb 2017 17:18:24 -0500 Subject: [PATCH 6/7] updated to create() --- import.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import.js b/import.js index b8613a3b9..8ca9b741b 100644 --- a/import.js +++ b/import.js @@ -13,7 +13,7 @@ var logger = require( 'pelias-logger' ).get( 'openaddresses' ); var parameters = require( './lib/parameters' ); var importPipeline = require( './lib/importPipeline' ); -const adminLookup = require('pelias-wof-admin-lookup').createLookupStream(); +const adminLookupStream = require('pelias-wof-admin-lookup'); var deduplicatorStream = require('./lib/streams/deduplicatorStream'); var addressDeduplicator = require('pelias-address-deduplicator'); @@ -47,5 +47,5 @@ if( 'exitCode' in args ){ var deduplicator = deduplicatorStream.create(peliasConfig, addressDeduplicator); - importPipeline.create( files, args.dirPath, deduplicator, adminLookup ); + importPipeline.create( files, args.dirPath, deduplicator, adminLookupStream.create() ); } From 6b1f544b8399b4d6f04567ef609129cf73dd6d6a Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Mon, 13 Feb 2017 15:24:27 -0500 Subject: [PATCH 7/7] updated to pelias-wof-admin-lookup 3.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 160770f44..45d365d51 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "pelias-dbclient": "2.0.0", "pelias-logger": "0.1.0", "pelias-model": "4.4.0", - "pelias-wof-admin-lookup": "pelias/wof-admin-lookup#incorporate-pip-service-directly", + "pelias-wof-admin-lookup": "3.0.0", "through2": "^2.0.3", "through2-filter": "^2.0.0", "through2-map": "^3.0.0",