From 2bd55ff17e8836715b8c248e068206abd0e05460 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 1 Feb 2019 16:51:58 -0800 Subject: [PATCH 1/2] PRC-996-2: Only update recently changed Applications. Fix application meta update logic to actually update the meta. Trivial update to make requests more clear. --- api/controllers/search.js | 6 +- api/helpers/utils.js | 72 ++- seed/shapesMigration/updateShapes.js | 707 ++++++++++++--------------- 3 files changed, 391 insertions(+), 394 deletions(-) diff --git a/api/controllers/search.js b/api/controllers/search.js index fb2eb51..f43d66b 100644 --- a/api/controllers/search.js +++ b/api/controllers/search.js @@ -79,7 +79,7 @@ exports.publicGetClientsInfoByDispositionId = function (args, res, next) { var searchURL = "http://maps.gov.bc.ca/arcgis/rest/services/mpcm/bcgw/MapServer/dynamicLayer/query?layer=%7B%22id%22%3A1%2C%22source%22%3A%7B%22type%22%3A%22dataLayer%22%2C%22dataSource%22%3A%7B%22type%22%3A%22table%22%2C%22workspaceId%22%3A%22MPCM_ALL_PUB%22%2C%22dataSourceName%22%3A%22WHSE_TANTALIS.TA_INTEREST_HOLDER_VW%22%7D%7D%7D&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&outSR=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&returnDistinctValues=false&f=json&where=DISPOSITION_TRANSACTION_SID="; return new Promise(function (resolve, reject) { - request({ url: searchURL + "'" + dtId + "'" }, function (err, res, body) { + request.get({ url: searchURL + "'" + dtId + "'" }, function (err, res, body) { if (err) { reject(err); } else if (res.statusCode !== 200) { @@ -119,7 +119,7 @@ exports.publicGetBCGW = function (args, res, next) { // var searchURL = "https://openmaps.gov.bc.ca/geo/pub/WHSE_TANTALIS.TA_CROWN_TENURES_SVW/ows?service=wfs&version=2.0.0&request=getfeature&typename=pub:WHSE_TANTALIS.TA_CROWN_TENURES_SVW&outputFormat=application/json&PROPERTYNAME=CROWN_LANDS_FILE&CQL_FILTER=CROWN_LANDS_FILE="; var searchURL = "https://openmaps.gov.bc.ca/geo/pub/WHSE_TANTALIS.TA_CROWN_TENURES_SVW/ows?service=wfs&version=2.0.0&request=getfeature&typename=PUB:WHSE_TANTALIS.TA_CROWN_TENURES_SVW&outputFormat=json&srsName=EPSG:4326&CQL_FILTER=CROWN_LANDS_FILE="; return new Promise(function (resolve, reject) { - request({ url: searchURL + "'" + clid + "'" }, function (err, res, body) { + request.get({ url: searchURL + "'" + clid + "'" }, function (err, res, body) { if (err) { reject(err); } else if (res.statusCode !== 200) { @@ -217,7 +217,7 @@ exports.publicGetBCGWDispositionTransactionId = function (args, res, next) { var searchURL = "https://openmaps.gov.bc.ca/geo/pub/WHSE_TANTALIS.TA_CROWN_TENURES_SVW/ows?service=wfs&version=2.0.0&request=getfeature&typename=PUB:WHSE_TANTALIS.TA_CROWN_TENURES_SVW&outputFormat=json&srsName=EPSG:4326&CQL_FILTER=DISPOSITION_TRANSACTION_SID="; return new Promise(function (resolve, reject) { - request({ url: searchURL + "'" + dtId + "'" }, function (err, res, body) { + request.get({ url: searchURL + "'" + dtId + "'" }, function (err, res, body) { if (err) { reject(err); } else if (res.statusCode !== 200) { diff --git a/api/helpers/utils.js b/api/helpers/utils.js index 1c56083..fa694ad 100644 --- a/api/helpers/utils.js +++ b/api/helpers/utils.js @@ -29,7 +29,7 @@ exports.buildQuery = function (property, values, query) { } return _.assignIn(query, { [property]: { $in: oids - } + } }); }; @@ -103,7 +103,7 @@ exports.runDataQuery = function (modelType, role, query, fields, sortWarmUp, sor _.each(defaultFields, function (f) { projection[f] = 1; }); - + // Add requested fields - sanitize first by including only those that we can/want to return _.each(fields, function (f) { projection[f] = 1; @@ -134,9 +134,9 @@ exports.runDataQuery = function (modelType, role, query, fields, sortWarmUp, sor } } }, - + sortWarmUp, // Used to setup the sort if a temporary projection is needed. - + !_.isEmpty(sort) ? { $sort: sort } : null, sort ? { $project: projection } : null, // Reset the projection just in case the sortWarmUp changed it. @@ -169,7 +169,7 @@ exports.runDataQuery = function (modelType, role, query, fields, sortWarmUp, sor exports.loginWebADE = function () { // Login to webADE and return access_token for use in subsequent calls. return new Promise(function (resolve, reject) { - request({ url: webADEAPI + "oauth/token?grant_type=client_credentials&disableDeveloperFilter=true", + request.get({ url: webADEAPI + "oauth/token?grant_type=client_credentials&disableDeveloperFilter=true", headers : { "Authorization" : "Basic " + new Buffer(username + ":" + password).toString("base64") } @@ -196,7 +196,7 @@ exports.loginWebADE = function () { exports.getApplicationByFilenumber = function (accessToken, clFile) { return new Promise(function (resolve, reject) { console.log("Looking up file:", _tantalisAPI + "landUseApplications?fileNumber=" + clFile); - request({ + request.get({ url: _tantalisAPI + "landUseApplications?fileNumber=" + clFile, auth: { bearer: accessToken @@ -244,7 +244,7 @@ exports.getApplicationByFilenumber = function (accessToken, clFile) { exports.getApplicationByDispositionID = function (accessToken, disp) { return new Promise(function (resolve, reject) { console.log("Looking up disposition:", _tantalisAPI + "landUseApplications/" + disp); - request({ + request.get({ url: _tantalisAPI + "landUseApplications/" + disp, auth: { bearer: accessToken @@ -364,4 +364,62 @@ exports.getApplicationByDispositionID = function (accessToken, disp) { } }); }); +}; + + +/** + * Fetches all application IDs from Tantalis given the params provided. + * + * @param {string} accessToken Tantalis API access token. (required) + * @param {object} [params={}] Object containing Tantalis query filters. (optional) + * @returns an array of matching Tantalis IDs. + */ +exports.getAllApplicationIDs = function (accessToken, params = {}) { + return new Promise(function (resolve, reject) { + // parse the optional params + var urlParams = ''; + _.forEach(params, function (value, key) { + urlParams += `${key}=${value}&`; + }) + urlParams = `?${urlParams.slice(0, -1)}` // Append an ? and remove the trailing & + + console.log("Looking up all applications:", _tantalisAPI + "landUseApplications" + urlParams); + + request.get({ + url: _tantalisAPI + "landUseApplications" + urlParams, + auth: { + bearer: accessToken + } + }, + function (err, res, body) { + if (err || (res && res.statusCode !== 200)) { + console.log("TTLS API ResponseCode:", err == null ? res.statusCode : err); + if (!err && res && res.statusCode) { + err = {}; + err.statusCode = res.statusCode; + } + reject(err); + } else { + try { + var applicationIDs = []; + + var response = JSON.parse(body); + _.forEach(response.elements, function (obj) { + if (obj) { + applicationIDs.push(obj.landUseApplicationId); + } + }); + if (applicationIDs.length) { + resolve(applicationIDs); + } else { + console.log("No applications found."); + resolve([]); + } + } catch (e) { + console.log("Object Parsing Failed:", e); + reject(e); + } + } + }); + }); }; \ No newline at end of file diff --git a/seed/shapesMigration/updateShapes.js b/seed/shapesMigration/updateShapes.js index bdb90b3..3d1da11 100644 --- a/seed/shapesMigration/updateShapes.js +++ b/seed/shapesMigration/updateShapes.js @@ -1,43 +1,43 @@ // // Example: node updateShapes.js admin admin https nrts-prc-dev.pathfinder.gov.bc.ca 443 // -var Promise = require('es6-promise').Promise; -var _ = require('lodash'); -var request = require('request'); -var querystring = require('querystring'); -var moment = require('moment'); -var Utils = require('../../api/helpers/utils'); -var Actions = require('../../api/helpers/actions'); -var username = ''; -var password = ''; -var protocol = 'http'; -var host = 'localhost'; -var port = '3000'; -var uri = ''; -var client_id = ''; -var grant_type = ''; -var auth_endpoint = 'http://localhost:3000/api/login/token'; -var _accessToken = ''; +var Promise = require('es6-promise').Promise; +var _ = require('lodash'); +var request = require('request'); +var querystring = require('querystring'); +var moment = require('moment'); +var Utils = require('../../api/helpers/utils'); +var Actions = require('../../api/helpers/actions'); +var username = ''; +var password = ''; +var protocol = 'http'; +var host = 'localhost'; +var port = '3000'; +var uri = ''; +var client_id = ''; +var grant_type = ''; +var auth_endpoint = 'http://localhost:3000/api/login/token'; +var _accessToken = ''; var args = process.argv.slice(2); if (args.length !== 8) { - console.log(''); - console.log('Please specify proper parameters: '); - console.log(''); - console.log('eg: node updateShapes.js admin admin http localhost 3000 client_id grant_type auth_endpoint'); - process.exit(1); - return; + console.log(''); + console.log('Please specify proper parameters: '); + console.log(''); + console.log('eg: node updateShapes.js admin admin http localhost 3000 client_id grant_type auth_endpoint'); + process.exit(1); + return; } else { - username = args[0]; - password = args[1]; - protocol = args[2]; - host = args[3]; - port = args[4]; - client_id = args[5]; - grant_type = args[6]; - auth_endpoint = args[7]; - uri = protocol + '://' + host + ':' + port + '/'; - console.log('Using connection:', uri); + username = args[0]; + password = args[1]; + protocol = args[2]; + host = args[3]; + port = args[4]; + client_id = args[5]; + grant_type = args[6]; + auth_endpoint = args[7]; + uri = protocol + '://' + host + ':' + port + '/'; + console.log('Using connection:', uri); } // JWT Login @@ -51,173 +51,142 @@ var jwt_login_time = null; // time we last logged in * @param {String} password * @returns {Promise} promise that resolves with the jwt_login token. */ -var login = function (username, password) { - return new Promise(function (resolve, reject) { - var body = querystring.stringify({ - grant_type: grant_type, - client_id: client_id, - username: username, - password: password - }); - var contentLength = body.length; - request.post({ - url: auth_endpoint, - headers: { - 'Content-Length': contentLength, - 'Content-Type': 'application/x-www-form-urlencoded' - }, - body: body - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - jwt_login = data.access_token; - jwt_expiry = data.expires_in; - jwt_login_time = new moment(); - resolve(data.access_token); - } - }); +var login = function(username, password) { + return new Promise(function(resolve, reject) { + var body = querystring.stringify({ + grant_type: grant_type, + client_id: client_id, + username: username, + password: password }); + var contentLength = body.length; + request.post( + { + url: auth_endpoint, + headers: { + 'Content-Length': contentLength, + 'Content-Type': 'application/x-www-form-urlencoded' + }, + body: body + }, + function(err, res, body) { + if (err || res.statusCode !== 200) { + console.log(' - login err:', err, res); + reject(null); + } else { + var data = JSON.parse(body); + jwt_login = data.access_token; + jwt_expiry = data.expires_in; + jwt_login_time = moment(); + resolve(data.access_token); + } + } + ); + }); }; /** - * Gets applications from ACRFD. + * Gets an application from ACRFD. * * @param {String} route the api route to call in the form: 'api/some/route'. (required) * @param {number} batchNumber the pagination page to return, starting at 0. (optional) * @param {number} batchSize the number of applications per page. (optional) * @returns {Promise} promise that resolves with an array of applications. */ -var getAllApplications = function (route, batchNumber=null, batchSize=null) { - return new Promise(function (resolve, reject) { - // only update the ones that aren't deleted - const url = uri + route + '?fields=tantalisID&isDeleted=false' + (batchNumber ? `&pageNum=${batchNumber}` : '') + (batchSize ? `&pageSize=${batchSize}` : ''); - console.log("Calling:", url); - request({ - url: url, headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - } - }, function (err, res, body) { - if (err) { - console.log("ERR:", err); - reject(err); - } else if (res.statusCode !== 200) { - console.log("res.statusCode:", res.statusCode); - reject(res.statusCode + ' ' + body); - } else { - var obj = {}; - try { - obj = JSON.parse(body); - console.log("Applications to process:", obj.length); - resolve(obj); - } catch (e) { - console.log("e:", e); - } - } - }); - }); +var getApplicationByID = function(route, tantalisID) { + return new Promise(function(resolve, reject) { + // only update the ones that aren't deleted + const url = uri + route + '?fields=tantalisID&isDeleted=false&tantalisId=' + tantalisID; + request( + { + url: url, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + } + }, + function(err, res, body) { + if (err) { + console.log(' - getApplication err:', err); + reject(err); + } else if (res.statusCode !== 200) { + console.log('res.statusCode:', res.statusCode); + reject(res.statusCode + ' ' + body); + } else { + var obj = {}; + try { + obj = JSON.parse(body); + resolve(obj); + } catch (e) { + console.log(' - getApplication parse err:', e); + } + } + } + ); + }); }; /** - * Get the total count of applications in ACRFD. + * Deletes the existing application features. * - * @param {String} route the api route to call in the form: 'api/some/route'. (required) - * @returns {number} count of applications. + * @param {Application} item Application + * @returns {Promise} */ -var getApplicationsCount = function (route) { - return new Promise(function (resolve, reject) { - // only update the ones that aren't deleted - const url = uri + route + '?isDeleted=false'; - console.log("Calling:", url); - request.head({ - url: url, headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - } - }, function (err, res, body) { - if (err) { - console.log("ERR:", err); - reject(err); - } else if (res.statusCode !== 200) { - console.log("res.statusCode:", res.statusCode); - reject(res.statusCode + ' ' + body); - } else { - try { - const count = parseInt(res.headers['x-total-count'], 10) - resolve(count); - } catch (e) { - console.log("e:", e); - } - } - }); +var deleteAllApplicationFeatures = function(item) { + return new Promise(function(resolve, reject) { + request.delete( + { + url: uri + 'api/feature?applicationID=' + item._id, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + } + }, + function(err, res, body) { + if (err || res.statusCode !== 200) { + console.log(' - deleteAllApplicationFeatures err:', err, res.body); + reject(null); + } else { + var data = JSON.parse(body); + resolve(data); + } + } + ); }); }; /** - * Updates the non-deleted applications in ACRFD. - * - Deletes old features for the application - * - Fetches the latest features and meta from Tantalis - * - Updates application features and meta. - * - Unpublishes the application if it is retired (statusHistoryEffectiveDate older than 6 months) + * Renews the jwt_login token if token expires soon. * - * @param {array} apps array of applications + * @returns {Promise} */ -var updateApplications = function(apps) { +var renewJWTLogin = function () { return new Promise(function (resolve, reject) { - Promise.resolve() - .then(function () { - return apps.reduce(function (current, item) { - return current.then(function () { - console.log("-------------------------------------------------------"); - console.log("Deleting existing features."); - // First delete all the application features. We blindly overwrite. - return deleteAllApplicationFeatures(item) - .then(function () { - // Fetch and store the features in the feature collection for this - // application. - console.log("Fetching and storing features for application ID:", item._id); - return getAndSaveFeatures(_accessToken, item); - }) - .then(function (app) { - if (app) { - // Update the application meta. - console.log("Updating application meta for DISP:", app.tantalisID); - // change this to reference the item data coming from TTLSAPI - return updateApplicationMeta(app); - } else { - // No feature - don't update meta. - console.log("No features found - not updating."); - return Promise.resolve(); - } - }) - .then(function (app) { - // If application is retired then unpublish it. - if (app && isRetired(app) && Actions.isPublished(app)) { - console.log("Application is now retired - UNPUBLISHING."); - return unpublishApplication(app); - } else { - return Promise.resolve(); - } - }); - }); - }, Promise.resolve()); - }).then(resolve, reject); + var duration = moment.duration(moment().diff(jwt_login_time)).asSeconds(); + // if less than 60 seconds left before token expiry. + if (duration > jwt_expiry - 60) { + console.log(' - Requesting new ACRFD login token.'); + return login(username, password).then(function () { + resolve(); + }); + } else { + resolve(); + } }); -} +}; /** * Updates and saves the application features. + * - * * @param {String} accessToken Tantalis api token - * @param {Application} item Application + * @param {Application} oldApp application as it exists in ACRFD + * @param {Application} newApp application with the latest values from Tantalis * @returns {Promise} */ var getAndSaveFeatures = function (accessToken, item) { - return new Promise(function (resolve, reject) { - Utils.getApplicationByDispositionID(accessToken, item.tantalisID) + return new Promise(function (resolve, reject) { + Utils.getApplicationByDispositionID(accessToken, item.tantalisID) .then(function (obj) { // console.log("returning:", obj); // Store the features in the DB @@ -228,54 +197,53 @@ var getAndSaveFeatures = function (accessToken, item) { var helpers = require('@turf/helpers'); var centroids = helpers.featureCollection([]); _.each(obj.parcels, function (f) { - // Tags default public - f.tags = [['sysadmin'], ['public']]; - // copy in all the app meta just to stay consistent. - f.properties.RESPONSIBLE_BUSINESS_UNIT = obj.RESPONSIBLE_BUSINESS_UNIT; - f.properties.TENURE_PURPOSE = obj.TENURE_PURPOSE; - f.properties.TENURE_SUBPURPOSE = obj.TENURE_SUBPURPOSE; - f.properties.TENURE_STATUS = obj.TENURE_STATUS; - f.properties.TENURE_TYPE = obj.TENURE_TYPE; - f.properties.TENURE_STAGE = obj.TENURE_STAGE; - f.properties.TENURE_SUBTYPE = obj.TENURE_SUBTYPE; - f.properties.TENURE_LOCATION = obj.TENURE_LOCATION; - f.properties.DISPOSITION_TRANSACTION_SID = obj.DISPOSITION_TRANSACTION_SID; - f.properties.CROWN_LANDS_FILE = obj.CROWN_LANDS_FILE; + // Tags default public + f.tags = [['sysadmin'], ['public']]; + // copy in all the app meta just to stay consistent. + f.properties.RESPONSIBLE_BUSINESS_UNIT = obj.RESPONSIBLE_BUSINESS_UNIT; + f.properties.TENURE_PURPOSE = obj.TENURE_PURPOSE; + f.properties.TENURE_SUBPURPOSE = obj.TENURE_SUBPURPOSE; + f.properties.TENURE_STATUS = obj.TENURE_STATUS; + f.properties.TENURE_TYPE = obj.TENURE_TYPE; + f.properties.TENURE_STAGE = obj.TENURE_STAGE; + f.properties.TENURE_SUBTYPE = obj.TENURE_SUBTYPE; + f.properties.TENURE_LOCATION = obj.TENURE_LOCATION; + f.properties.DISPOSITION_TRANSACTION_SID = obj.DISPOSITION_TRANSACTION_SID; + f.properties.CROWN_LANDS_FILE = obj.CROWN_LANDS_FILE; - allFeaturesForDisp.push(f); - // Get the polygon and put it for later centroid calculation - centroids.features.push(turf.centroid(f)); + allFeaturesForDisp.push(f); + // Get the polygon and put it for later centroid calculation + centroids.features.push(turf.centroid(f)); }); // Centroid of all the shapes. - var featureCollectionCentroid; if (centroids.features.length > 0) { - item.centroid = turf.centroid(centroids).geometry.coordinates; + item.centroid = turf.centroid(centroids).geometry.coordinates; } item.client = ""; for (let [idx, client] of Object.entries(obj.interestedParties)) { - if (idx > 0) { - item.client += ", "; - } - if (client.interestedPartyType == 'O') { - item.client += client.legalName; - } else { - item.client += client.firstName + " " + client.lastName; - } + if (idx > 0) { + item.client += ", "; + } + if (client.interestedPartyType == 'O') { + item.client += client.legalName; + } else { + item.client += client.firstName + " " + client.lastName; + } } item.statusHistoryEffectiveDate = obj.statusHistoryEffectiveDate; Promise.resolve() - .then(function () { + .then(function () { return allFeaturesForDisp.reduce(function (previousItem, currentItem) { - return previousItem.then(function () { - return doFeatureSave(currentItem, item._id); - }); + return previousItem.then(function () { + return doFeatureSave(currentItem, item._id); + }); }, Promise.resolve()); - }).then(function () { - resolve(item); - }); + }).then(function () { + resolve({ app: item, tantalisApp: obj }); + }); }); - }); + }); }; /** @@ -285,52 +253,29 @@ var getAndSaveFeatures = function (accessToken, item) { * @param {String} appId Application id * @returns {Promise} */ -var doFeatureSave = function (item, appId) { - return new Promise(function (resolve, reject) { - item.applicationID = appId; - request.post({ - url: uri + 'api/feature', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(item) - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); -}; - -/** - * Deletes the existing application features. - * - * @param {Application} item Application - * @returns {Promise} - */ -var deleteAllApplicationFeatures = function (item) { - return new Promise(function (resolve, reject) { - request.delete({ - url: uri + 'api/feature?applicationID=' + item._id, - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); +var doFeatureSave = function(item, appId) { + return new Promise(function(resolve, reject) { + item.applicationID = appId; + request.post( + { + url: uri + 'api/feature', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + }, + body: JSON.stringify(item) + }, + function(err, res, body) { + if (err || res.statusCode !== 200) { + console.log(' - doFeatureSave err:', err, res); + reject(null); + } else { + var data = JSON.parse(body); + resolve(data); + } + } + ); + }); }; /** @@ -339,39 +284,40 @@ var deleteAllApplicationFeatures = function (item) { * @param {Application} app Application * @returns {Promise} */ -var updateApplicationMeta = function (item) { - return new Promise(function (resolve, reject) { - var updatedAppObject = {}; - updatedAppObject.businessUnit = item.RESPONSIBLE_BUSINESS_UNIT; - updatedAppObject.purpose = item.TENURE_PURPOSE; - updatedAppObject.subpurpose = item.TENURE_SUBPURPOSE; - updatedAppObject.status = item.TENURE_STATUS; - updatedAppObject.type = item.TENURE_TYPE; - updatedAppObject.tenureStage = item.TENURE_STAGE; - updatedAppObject.subtype = item.TENURE_SUBTYPE; - updatedAppObject.location = item.TENURE_LOCATION; - updatedAppObject.legalDescription = item.TENURE_LEGAL_DESCRIPTION; - updatedAppObject.centroid = item.centroid; - updatedAppObject.areaHectares = item.areaHectares; - updatedAppObject.client = item.client; - updatedAppObject.statusHistoryEffectiveDate = item.statusHistoryEffectiveDate; - request.put({ - url: uri + 'api/application/' + item._id, - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(updatedAppObject), - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); +var updateApplicationMeta = function (item, tantalisItem) { + return new Promise(function (resolve, reject) { + var updatedAppObject = {}; + updatedAppObject.businessUnit = tantalisItem.RESPONSIBLE_BUSINESS_UNIT; + updatedAppObject.purpose = tantalisItem.TENURE_PURPOSE; + updatedAppObject.subpurpose = tantalisItem.TENURE_SUBPURPOSE; + updatedAppObject.status = tantalisItem.TENURE_STATUS; + updatedAppObject.type = tantalisItem.TENURE_TYPE; + updatedAppObject.tenureStage = tantalisItem.TENURE_STAGE; + updatedAppObject.subtype = tantalisItem.TENURE_SUBTYPE; + updatedAppObject.location = tantalisItem.TENURE_LOCATION; + updatedAppObject.legalDescription = tantalisItem.TENURE_LEGAL_DESCRIPTION; + updatedAppObject.centroid = item.centroid; + updatedAppObject.areaHectares = item.areaHectares; + updatedAppObject.client = item.client; + updatedAppObject.statusHistoryEffectiveDate = item.statusHistoryEffectiveDate; + + request.put({ + url: uri + 'api/application/' + item._id, + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + jwt_login + }, + body: JSON.stringify(updatedAppObject), + }, function (err, res, body) { + if (err || res.statusCode !== 200) { + console.log(" - updateApplicationMeta err:", err, res); + reject(null); + } else { + var data = JSON.parse(body); + resolve(data); + } }); + }); }; /** @@ -380,7 +326,7 @@ var updateApplicationMeta = function (item) { * @param {Application} app * @returns {Boolean} True if the application is retired, false otherwise. */ -var isRetired = function (app) { +var isRetired = function(app) { if (app.status) { // check if retired status let isRetiredStatus = false; @@ -427,115 +373,108 @@ var isRetired = function (app) { */ var unpublishApplication = function (app) { return new Promise(function (resolve, reject) { - request.put({ - url: uri + 'api/application/' + app._id + '/unpublish', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(app), - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); + request.put({ + url: uri + 'api/application/' + app._id + '/unpublish', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + }, + body: JSON.stringify(app) + }, + function (err, res, body) { + if (err || res.statusCode !== 200) { + console.log(' - unpublishApplication err:', err, res); + reject(null); + } else { + var data = JSON.parse(body); + resolve(data); + } + } + ); }); }; +console.log('======================================================='); +console.log('1. Authenticating with ACRFD.'); /** - * Renews the jwt_login token if necessary. - * - * @returns {Promise} - */ -var renewJWTLogin = function () { - return new Promise(function(resolve, reject) { - var duration = moment.duration(new moment().diff(jwt_login_time)).asSeconds(); - // less than 180 seconds left before token expiry - if (duration > jwt_expiry - 180) { - console.log('Requesting new login token'); - return login(username, password) - .then(function () { - resolve() - }) - } else { - resolve(); - } - }); -} - -/** - * Returns an array of objects containing the optional batch parameters used by #getAllApplications. - * @param {number} applicationsCount - * @returns {array} batches array of objects used to facilitate calling #getAllApplications in batches. - */ -var getBatches = function (applicationsCount) { - var batches = []; - - var batchNumber = 0; - const batchSize = 100; - while(applicationsCount > 0) { - batches.push({ - batchNumber: batchNumber, - batchSize: Math.min(batchSize, applicationsCount) - }) - - batchNumber += 1; - applicationsCount -= batchSize; - } - - return Promise.resolve(batches); -} - -console.log("Logging in and getting JWT."); - -/** - * Main call that updates applications. + * Main call chain that utilizes the above functions to update ACRFD applications. */ +// Authenticate to ACRFD and get jwt token login(username, password) .then(function () { - // Get a token from webade for TTLS API calls (getAndSaveFeatures) - return Utils.loginWebADE() - .then(function (accessToken) { - console.log("TTLS API login token:", accessToken); - _accessToken = accessToken; - return _accessToken; - }); - }) - .then(function () { - console.log("Getting applications count."); - return getApplicationsCount('api/application'); - }) - .then(function (applicationsCount) { - console.log(`Applications count: ${applicationsCount}`); - return getBatches(applicationsCount); - }) - .then(function (batches) { - console.log(batches); - return batches.reduce(function(previousItem, currentItem){ - return previousItem - .then(function(){ - console.log("-------------------------------------------------------"); - return renewJWTLogin() // Each batch iteration, check if the login token needs to be re-fetched. - .then(function() { - console.log(`Getting applications - batch ${currentItem.batchNumber}.`); - return getAllApplications('api/application', currentItem.batchNumber, currentItem.batchSize) + // Authenticate to Tantalis and get access token used by #getAndSaveFeatures + console.log('2. Authenticating with Tantalis.'); + return Utils.loginWebADE().then(function (accessToken) { + console.log(' - TTLS API login token:', accessToken); + _accessToken = accessToken; + return _accessToken; + }); + }) + .then(function () { + console.log('3. Fetching all Tantalis applications that have been updated in the last day.'); + // Get applications from Tantalis that have been updated in the last day + var lastDay = moment().subtract(1, 'days').format('YYYYMMDD'); + return Utils.getAllApplicationIDs(_accessToken, { updated: lastDay }); + }) + .then(function (recentlyUpdatedApplicationIDs) { + console.log(` - Found ${recentlyUpdatedApplicationIDs.length} recently updated Tantalis Applications.`); + // For each application from Tantalis, fetch the matching record in ACRFD if it exists and update it + return recentlyUpdatedApplicationIDs.reduce(function (previousItem, currentItem) { + return previousItem.then(function () { + console.log('-----------------------------------------------'); + // Each iteration, check if the login token needs to be re-fetched + return renewJWTLogin() + .then(function () { + console.log(`4. Attempting to find matching ACRFD Application, tantalisID: ${currentItem}`); + // Attempt to fetch the ACRFD application, which may not exist + return getApplicationByID('api/application', currentItem); }) .then(function (apps) { - // Now iterate through each application, grabbing the tantalisID and populating the shapes in the feature collection. - return updateApplications(apps); + // We really only expect 1 result, but the API returns an array + return apps.reduce(function (previousItem, currentItem) { + return previousItem.then(function () { + console.log(`5. Matching ACRFD Application found`); + console.log(" - Deleting existing application features"); + // First delete all the application features. We blindly overwrite. + return deleteAllApplicationFeatures(currentItem) + .then(function () { + console.log(" - Updating new application features"); + // Fetch and store the features in the feature collection for this application. + return getAndSaveFeatures(_accessToken, currentItem); + }) + .then(function ({ app, tantalisApp }) { + if (app && tantalisApp) { + console.log(" - Updating new application meta"); + // Update the application meta. + return updateApplicationMeta(app, tantalisApp); + } else { + console.log(" - No features found - not updating."); + // No features.on't update meta. + return Promise.resolve(); + } + }) + .then(function (app) { + // If application is retired then unpublish it. + if (app && isRetired(app) && Actions.isPublished(app)) { + console.log(" - Application is now retired - UNPUBLISHING."); + return unpublishApplication(app); + } else { + return Promise.resolve(); + } + }); + }); + }, Promise.resolve()); }); - }) - }, Promise.resolve()); - }) - .then(function () { - console.log("-------------------------------------------------------"); - console.log("Done!"); - }) - .catch(function (err) { - console.log("ERR:", err); - process.exit(1); - }); \ No newline at end of file + }); + }, + Promise.resolve()); + }) + .then(function () { + console.log('-----------------------------------------------'); + console.log('Done!'); + console.log('======================================================='); + }) + .catch(function (err) { + console.log(' - general err:', err); + process.exit(1); + }); \ No newline at end of file From bc1d981b59194e7eb602ff18b2c51101d6b4e20c Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 5 Feb 2019 12:47:18 -0800 Subject: [PATCH 2/2] Update to use qs. --- api/helpers/utils.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/api/helpers/utils.js b/api/helpers/utils.js index fa694ad..4701d1e 100644 --- a/api/helpers/utils.js +++ b/api/helpers/utils.js @@ -3,7 +3,7 @@ var _ = require('lodash'); var mongoose = require('mongoose'); var clamav = require('clamav.js'); -var fs = require('fs'); +var qs = require('qs'); var request = require('request'); var turf = require('@turf/turf'); var helpers = require('@turf/helpers'); @@ -368,25 +368,20 @@ exports.getApplicationByDispositionID = function (accessToken, disp) { /** - * Fetches all application IDs from Tantalis given the params provided. + * Fetches all application IDs from Tantalis given the filter params provided. * * @param {string} accessToken Tantalis API access token. (required) - * @param {object} [params={}] Object containing Tantalis query filters. (optional) + * @param {object} [filterParams={}] Object containing Tantalis query filters. See Tantalis API Spec. (optional) * @returns an array of matching Tantalis IDs. */ -exports.getAllApplicationIDs = function (accessToken, params = {}) { +exports.getAllApplicationIDs = function (accessToken, filterParams = {}) { return new Promise(function (resolve, reject) { - // parse the optional params - var urlParams = ''; - _.forEach(params, function (value, key) { - urlParams += `${key}=${value}&`; - }) - urlParams = `?${urlParams.slice(0, -1)}` // Append an ? and remove the trailing & + const queryString = `?${qs.stringify(filterParams)}`; - console.log("Looking up all applications:", _tantalisAPI + "landUseApplications" + urlParams); + console.log("Looking up all applications:", _tantalisAPI + "landUseApplications" + queryString); request.get({ - url: _tantalisAPI + "landUseApplications" + urlParams, + url: _tantalisAPI + "landUseApplications" + queryString, auth: { bearer: accessToken } @@ -394,6 +389,7 @@ exports.getAllApplicationIDs = function (accessToken, params = {}) { function (err, res, body) { if (err || (res && res.statusCode !== 200)) { console.log("TTLS API ResponseCode:", err == null ? res.statusCode : err); + console.log("TTLS API ResponseCode:", body); if (!err && res && res.statusCode) { err = {}; err.statusCode = res.statusCode;