diff --git a/api/helpers/models/application.js b/api/helpers/models/application.js index b5467bd..4817095 100644 --- a/api/helpers/models/application.js +++ b/api/helpers/models/application.js @@ -20,7 +20,7 @@ status : { type: String }, subpurpose : { type: String }, subtype : { type: String }, - tantalisID : { type: Number, unique: true, default: 0 }, + tantalisID : { type: Number, default: 0 }, tenureStage : { type: String }, type : { type: String }, diff --git a/api/helpers/utils.js b/api/helpers/utils.js index 4701d1e..2a17425 100644 --- a/api/helpers/utils.js +++ b/api/helpers/utils.js @@ -368,7 +368,7 @@ exports.getApplicationByDispositionID = function (accessToken, disp) { /** - * Fetches all application IDs from Tantalis given the filter params provided. + * Fetches all application landUseApplicationIds (aka: dispositionID, tantalisID) from Tantalis given the filter params provided. * * @param {string} accessToken Tantalis API access token. (required) * @param {object} [filterParams={}] Object containing Tantalis query filters. See Tantalis API Spec. (optional) diff --git a/seed/shapesMigration/updateAllShapes.js b/seed/shapesMigration/updateAllShapes.js deleted file mode 100644 index 9e1589a..0000000 --- a/seed/shapesMigration/updateAllShapes.js +++ /dev/null @@ -1,549 +0,0 @@ -/** - * This is a performance heavy version of updateShapes.js that blindly updates ALL ACRFD applications, and should only be used sparingly as needed. - * - * Fetches all ACRFD applications, in batches of 100. - * For each application, fetches the matching Tantalis application, and updates the ACRDF application features and meta to match whatever is in Tantalis (the source of truth). - * - * Additionally, if an ACRFD application has reach a retired state, it unpublishes the application. - */ -var Promise = require('es6-promise').Promise; -var _ = require('lodash'); -var request = require('request'); -var querystring = require('querystring'); -var moment = require('moment'); -var Utils = require('../../api/helpers/utils'); -var Actions = require('../../api/helpers/actions'); -var username = ''; -var password = ''; -var protocol = 'http'; -var host = 'localhost'; -var port = '3000'; -var uri = ''; -var client_id = ''; -var grant_type = ''; -var auth_endpoint = 'http://localhost:3000/api/login/token'; -var _accessToken = ''; - -var args = process.argv.slice(2); -if (args.length !== 8) { - console.log(''); - console.log('Please specify proper parameters: '); - console.log(''); - console.log('eg: node updateShapes.js admin admin http localhost 3000 client_id grant_type auth_endpoint'); - process.exit(1); - return; -} else { - username = args[0]; - password = args[1]; - protocol = args[2]; - host = args[3]; - port = args[4]; - client_id = args[5]; - grant_type = args[6]; - auth_endpoint = args[7]; - uri = protocol + '://' + host + ':' + port + '/'; - console.log('Using connection:', uri); -} - -// JWT Login -var jwt_login = null; // the ACRFD login token -var jwt_expiry = null; // how long the token lasts before expiring -var jwt_login_time = null; // time we last logged in -/** - * Logs in to ACRFD. - * - * @param {String} username - * @param {String} password - * @returns {Promise} promise that resolves with the jwt_login token. - */ -var login = function (username, password) { - return new Promise(function (resolve, reject) { - var body = querystring.stringify({ - grant_type: grant_type, - client_id: client_id, - username: username, - password: password - }); - var contentLength = body.length; - request.post({ - url: auth_endpoint, - headers: { - 'Content-Length': contentLength, - 'Content-Type': 'application/x-www-form-urlencoded' - }, - body: body - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - jwt_login = data.access_token; - jwt_expiry = data.expires_in; - jwt_login_time = new moment(); - resolve(data.access_token); - } - }); - }); -}; - -/** - * Gets applications from ACRFD. - * - * @param {String} route the api route to call in the form: 'api/some/route'. (required) - * @param {number} batchNumber the pagination page to return, starting at 0. (optional) - * @param {number} batchSize the number of applications per page. (optional) - * @returns {Promise} promise that resolves with an array of applications. - */ -var getAllApplications = function (route, batchNumber=null, batchSize=null) { - return new Promise(function (resolve, reject) { - // only update the ones that aren't deleted - const url = uri + route + '?fields=tantalisID&isDeleted=false' + (batchNumber ? `&pageNum=${batchNumber}` : '') + (batchSize ? `&pageSize=${batchSize}` : ''); - console.log("Calling:", url); - request.get({ - url: url, headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - } - }, function (err, res, body) { - if (err) { - console.log("ERR:", err); - reject(err); - } else if (res.statusCode !== 200) { - console.log("res.statusCode:", res.statusCode); - reject(res.statusCode + ' ' + body); - } else { - var obj = {}; - try { - obj = JSON.parse(body); - console.log("Applications to process:", obj.length); - resolve(obj); - } catch (e) { - console.log("e:", e); - } - } - }); - }); -}; - -/** - * Get the total count of applications in ACRFD. - * - * @param {String} route the api route to call in the form: 'api/some/route'. (required) - * @returns {number} count of applications. - */ -var getApplicationsCount = function (route) { - return new Promise(function (resolve, reject) { - // only update the ones that aren't deleted - const url = uri + route + '?isDeleted=false'; - console.log("Calling:", url); - request.head({ - url: url, headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - } - }, function (err, res, body) { - if (err) { - console.log("ERR:", err); - reject(err); - } else if (res.statusCode !== 200) { - console.log("res.statusCode:", res.statusCode); - reject(res.statusCode + ' ' + body); - } else { - try { - const count = parseInt(res.headers['x-total-count'], 10) - resolve(count); - } catch (e) { - console.log("e:", e); - } - } - }); - }); -}; - -/** - * Updates the non-deleted applications in ACRFD. - * - Deletes old features for the application - * - Fetches the latest features and meta from Tantalis - * - Updates application features and meta. - * - Unpublishes the application if it is retired (statusHistoryEffectiveDate older than 6 months) - * - * @param {array} apps array of applications - */ -var updateApplications = function (apps) { - return new Promise(function (resolve, reject) { - Promise.resolve() - .then(function () { - return apps.reduce(function (current, item) { - return current.then(function () { - // Each iteration, check if the login token needs to be re-fetched. - return renewJWTLogin(); - }).then(function () { - console.log("-------------------------------------------------------"); - console.log("Deleting existing features."); - // First delete all the application features. We blindly overwrite. - return deleteAllApplicationFeatures(item) - .then(function () { - // Fetch and store the features in the feature collection for this - // application. - console.log("Fetching and storing features for application ID:", item._id); - return getAndSaveFeatures(_accessToken, item); - }) - .then(function (app) { - if (app) { - // Update the application meta. - console.log("Updating application meta for DISP:", app.tantalisID); - // change this to reference the item data coming from TTLSAPI - return updateApplicationMeta(app); - } else { - // No feature - don't update meta. - console.log("No features found - not updating."); - return Promise.resolve(); - } - }) - .then(function (app) { - // If application is retired then unpublish it. - if (app && isRetired(app) && Actions.isPublished(app)) { - console.log("Application is now retired - UNPUBLISHING."); - return unpublishApplication(app); - } else { - return Promise.resolve(); - } - }); - }); - }, Promise.resolve()); - }).then(resolve, reject); - }); -} - -/** - * Updates and saves the application features. - * - * @param {String} accessToken Tantalis api token - * @param {Application} item Application - * @returns {Promise} - */ -var getAndSaveFeatures = function (accessToken, item) { - return new Promise(function (resolve, reject) { - Utils.getApplicationByDispositionID(accessToken, item.tantalisID) - .then(function (obj) { - // console.log("returning:", obj); - // Store the features in the DB - var allFeaturesForDisp = []; - item.areaHectares = obj.areaHectares; - - var turf = require('@turf/turf'); - var helpers = require('@turf/helpers'); - var centroids = helpers.featureCollection([]); - _.each(obj.parcels, function (f) { - // Tags default public - f.tags = [['sysadmin'], ['public']]; - // copy in all the app meta just to stay consistent. - f.properties.RESPONSIBLE_BUSINESS_UNIT = obj.RESPONSIBLE_BUSINESS_UNIT; - f.properties.TENURE_PURPOSE = obj.TENURE_PURPOSE; - f.properties.TENURE_SUBPURPOSE = obj.TENURE_SUBPURPOSE; - f.properties.TENURE_STATUS = obj.TENURE_STATUS; - f.properties.TENURE_TYPE = obj.TENURE_TYPE; - f.properties.TENURE_STAGE = obj.TENURE_STAGE; - f.properties.TENURE_SUBTYPE = obj.TENURE_SUBTYPE; - f.properties.TENURE_LOCATION = obj.TENURE_LOCATION; - f.properties.DISPOSITION_TRANSACTION_SID = obj.DISPOSITION_TRANSACTION_SID; - f.properties.CROWN_LANDS_FILE = obj.CROWN_LANDS_FILE; - - allFeaturesForDisp.push(f); - // Get the polygon and put it for later centroid calculation - centroids.features.push(turf.centroid(f)); - }); - // Centroid of all the shapes. - var featureCollectionCentroid; - if (centroids.features.length > 0) { - item.centroid = turf.centroid(centroids).geometry.coordinates; - } - item.client = ""; - for (let [idx, client] of Object.entries(obj.interestedParties)) { - if (idx > 0) { - item.client += ", "; - } - if (client.interestedPartyType == 'O') { - item.client += client.legalName; - } else { - item.client += client.firstName + " " + client.lastName; - } - } - item.statusHistoryEffectiveDate = obj.statusHistoryEffectiveDate; - - Promise.resolve() - .then(function () { - return allFeaturesForDisp.reduce(function (previousItem, currentItem) { - return previousItem.then(function () { - return doFeatureSave(currentItem, item._id); - }); - }, Promise.resolve()); - }).then(function () { - resolve(item); - }); - }); - }); -}; - -/** - * Updates and saves the application features. - * - * @param {Application} item Application - * @param {String} appId Application id - * @returns {Promise} - */ -var doFeatureSave = function (item, appId) { - return new Promise(function (resolve, reject) { - item.applicationID = appId; - request.post({ - url: uri + 'api/feature', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(item) - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); -}; - -/** - * Deletes the existing application features. - * - * @param {Application} item Application - * @returns {Promise} - */ -var deleteAllApplicationFeatures = function (item) { - return new Promise(function (resolve, reject) { - request.delete({ - url: uri + 'api/feature?applicationID=' + item._id, - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); -}; - -/** - * Updates and saves the application meta. - * - * @param {Application} app Application - * @returns {Promise} - */ -var updateApplicationMeta = function (item) { - return new Promise(function (resolve, reject) { - var updatedAppObject = {}; - updatedAppObject.businessUnit = item.RESPONSIBLE_BUSINESS_UNIT; - updatedAppObject.purpose = item.TENURE_PURPOSE; - updatedAppObject.subpurpose = item.TENURE_SUBPURPOSE; - updatedAppObject.status = item.TENURE_STATUS; - updatedAppObject.type = item.TENURE_TYPE; - updatedAppObject.tenureStage = item.TENURE_STAGE; - updatedAppObject.subtype = item.TENURE_SUBTYPE; - updatedAppObject.location = item.TENURE_LOCATION; - updatedAppObject.legalDescription = item.TENURE_LEGAL_DESCRIPTION; - updatedAppObject.centroid = item.centroid; - updatedAppObject.areaHectares = item.areaHectares; - updatedAppObject.client = item.client; - updatedAppObject.statusHistoryEffectiveDate = item.statusHistoryEffectiveDate; - request.put({ - url: uri + 'api/application/' + item._id, - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(updatedAppObject), - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); -}; - -/** - * Returns whether or not the application is retired or not (statusHistoryEffectiveDate older than 6 months). - * - * @param {Application} app - * @returns {Boolean} True if the application is retired, false otherwise. - */ -var isRetired = function (app) { - if (app.status) { - // check if retired status - let isRetiredStatus = false; - switch (app.status.toUpperCase()) { - case 'ABANDONED': - case 'CANCELLED': - case 'OFFER NOT ACCEPTED': - case 'OFFER RESCINDED': - case 'RETURNED': - case 'REVERTED': - case 'SOLD': - case 'SUSPENDED': - case 'WITHDRAWN': - isRetiredStatus = true; // ABANDONED - break; - - case 'ACTIVE': - case 'COMPLETED': - case 'DISPOSITION IN GOOD STANDING': - case 'EXPIRED': - case 'HISTORIC': - isRetiredStatus = true; // APPROVED - break; - - case 'DISALLOWED': - isRetiredStatus = true; // NOT APPROVED - break; - } - - if (isRetiredStatus) { - // check if retired more than 6 months ago - return moment(app.statusHistoryEffectiveDate).endOf('day').add(6, 'months').isBefore(); - } - } - - return false; -}; - -/** - * Unpublishes the application. - * - * @param {APplication} app - * @returns {Promise} - */ -var unpublishApplication = function (app) { - return new Promise(function (resolve, reject) { - request.put({ - url: uri + 'api/application/' + app._id + '/unpublish', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login - }, - body: JSON.stringify(app), - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log("err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); - } - }); - }); -}; - -/** - * Renews the jwt_login token if necessary. - * - * @returns {Promise} - */ -var renewJWTLogin = function () { - return new Promise(function(resolve, reject) { - var duration = moment.duration(new moment().diff(jwt_login_time)).asSeconds(); - // less than 60 seconds left before token expiry - if (duration > jwt_expiry - 60) { - console.log('Requesting new login token'); - return login(username, password) - .then(function () { - resolve() - }) - } else { - resolve(); - } - }); -} - -/** - * Returns an array of objects containing the optional batch parameters used by #getAllApplications. - * @param {number} applicationsCount - * @returns {array} batches array of objects used to facilitate calling #getAllApplications in batches. - */ -var getBatches = function (applicationsCount) { - var batches = []; - - var batchNumber = 0; - const batchSize = 100; - while(applicationsCount > 0) { - batches.push({ - batchNumber: batchNumber, - batchSize: Math.min(batchSize, applicationsCount) - }) - - batchNumber += 1; - applicationsCount -= batchSize; - } - - return Promise.resolve(batches); -} - -console.log("Logging in and getting JWT."); - -/** - * Main call that updates applications. - */ -login(username, password) - .then(function () { - // Get a token from webade for TTLS API calls (getAndSaveFeatures) - return Utils.loginWebADE() - .then(function (accessToken) { - console.log("TTLS API login token:", accessToken); - _accessToken = accessToken; - return _accessToken; - }); - }) - .then(function () { - console.log("Getting applications count."); - return getApplicationsCount('api/application'); - }) - .then(function (applicationsCount) { - console.log(`Applications count: ${applicationsCount}`); - return getBatches(applicationsCount); - }) - .then(function (batches) { - console.log(batches); - return batches.reduce(function(previousItem, currentItem){ - return previousItem - .then(function(){ - console.log("-------------------------------------------------------"); - return renewJWTLogin() // Each batch iteration, check if the login token needs to be re-fetched. - .then(function() { - console.log(`Getting applications - batch ${currentItem.batchNumber}.`); - return getAllApplications('api/application', currentItem.batchNumber, currentItem.batchSize) - }) - .then(function (apps) { - // Now iterate through each application, grabbing the tantalisID and populating the shapes in the feature collection. - return updateApplications(apps); - }); - }) - }, Promise.resolve()); - }) - .then(function () { - console.log("-------------------------------------------------------"); - console.log("Done!"); - }) - .catch(function (err) { - console.log("ERR:", err); - process.exit(1); - }); \ No newline at end of file diff --git a/seed/shapesMigration/updateShapes.js b/seed/shapesMigration/updateShapes.js index 7c768b0..cc00e11 100644 --- a/seed/shapesMigration/updateShapes.js +++ b/seed/shapesMigration/updateShapes.js @@ -1,9 +1,15 @@ /** - * Fetches all ACRFD applications that may have reach a retired state, and unpublishes any found. + * This script performs various updates to ACRFD applications in order to keep them up to date with whatever information is in Tantalis (the source of truth). * - * Fetches all Tantalis applications that have had an update within the last day. - * For each Tantalis application, attempts to find a matching ACRFD application. - * If one exists, updates the features and meta to match whatever is in Tantalis (the source of truth). + * 1. Authenticates with ACRFD + * 2. Unpublishes retired applications: + * a. Fetches all ACRFD applications that may have reached a retired state, and unpublishes any found. + * 3. AUthenticates with Tantalis + * 4. Updates non-deleted ACRFD applications: + * a. Fetches all Tantalis applications that have had an update within the last 1 day. + * b. Fetches all non-deleted ACRFD tantalisIDs. + * c. For each ACRFD application with a matching Tantalis application: + * i. Updates the ACRFD application features and meta to match whatever is in Tantalis (the source of truth). */ var Promise = require('es6-promise').Promise; var _ = require('lodash'); @@ -24,11 +30,11 @@ var auth_endpoint = 'http://localhost:3000/api/login/token'; var _accessToken = ''; var args = process.argv.slice(2); +console.log('======================================================='); if (args.length !== 8) { - console.log(''); console.log('Please specify proper parameters: '); - console.log(''); - console.log('eg: node updateShapes.js admin admin http localhost 3000 client_id grant_type auth_endpoint'); + console.log('Example: node updateShapes.js admin admin http localhost 3000 client_id grant_type auth_endpoint'); + console.log('======================================================='); process.exit(1); return; } else { @@ -42,12 +48,19 @@ if (args.length !== 8) { auth_endpoint = args[7]; uri = protocol + '://' + host + ':' + port + '/'; console.log('Using connection:', uri); + console.log('-----------------------------------------------'); } -// JWT Login +// Used when unpublishing retired applications. +var retiredStatuses = ['ABANDONED', 'CANCELLED', 'OFFER NOT ACCEPTED', 'OFFER RESCINDED', 'RETURNED', 'REVERTED', 'SOLD', +'SUSPENDED', 'WITHDRAWN', 'ACTIVE', 'COMPLETED', 'DISPOSITION IN GOOD STANDING', 'EXPIRED', 'HISTORIC', 'DISALLOWED']; + + +// Used to renew the ACRFD login tokes before it expires if the update script takes longer than the lifespan of the token. var jwt_login = null; // the ACRFD login token var jwt_expiry = null; // how long the token lasts before expiring var jwt_login_time = null; // time we last logged in + /** * Logs in to ACRFD. * @@ -55,8 +68,8 @@ var jwt_login_time = null; // time we last logged in * @param {String} password * @returns {Promise} promise that resolves with the jwt_login token. */ -var loginToACRFD = function (username, password) { - return new Promise(function (resolve, reject) { +var loginToACRFD = function(username, password) { + return new Promise(function(resolve, reject) { var body = querystring.stringify({ grant_type: grant_type, client_id: client_id, @@ -64,7 +77,8 @@ var loginToACRFD = function (username, password) { password: password }); var contentLength = body.length; - request.post({ + request.post( + { url: auth_endpoint, headers: { 'Content-Length': contentLength, @@ -72,7 +86,7 @@ var loginToACRFD = function (username, password) { }, body: body }, - function (err, res, body) { + function(err, res, body) { if (err || res.statusCode !== 200) { console.log(' - Login err:', err, res); reject(null); @@ -96,18 +110,19 @@ var loginToACRFD = function (username, password) { * @param {number} batchSize the number of applications per page. (optional) * @returns {Promise} promise that resolves with an array of applications. */ -var getApplicationByID = function (route, tantalisID) { - return new Promise(function (resolve, reject) { +var getApplicationByID = function(route, tantalisID) { + return new Promise(function(resolve, reject) { // only update the ones that aren't deleted const url = uri + route + '?fields=tantalisID&isDeleted=false&tantalisId=' + tantalisID; - request.get({ + request.get( + { url: url, headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + jwt_login } }, - function (err, res, body) { + function(err, res, body) { if (err) { console.log(' - getApplication err:', err); reject(err); @@ -131,19 +146,20 @@ var getApplicationByID = function (route, tantalisID) { /** * Deletes the existing application features. * - * @param {Application} acrfdItem Application + * @param {Application} acrfdApp Application * @returns {Promise} */ -var deleteAllApplicationFeatures = function (acrfdItem) { - return new Promise(function (resolve, reject) { - request.delete({ - url: uri + 'api/feature?applicationID=' + acrfdItem._id, +var deleteAllApplicationFeatures = function(acrfdApp) { + return new Promise(function(resolve, reject) { + request.delete( + { + url: uri + 'api/feature?applicationID=' + acrfdApp._id, headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + jwt_login } }, - function (err, res, body) { + function(err, res, body) { if (err || res.statusCode !== 200) { console.log(' - deleteAllApplicationFeatures err:', err, res.body); reject(null); @@ -161,13 +177,13 @@ var deleteAllApplicationFeatures = function (acrfdItem) { * * @returns {Promise} */ -var renewJWTLogin = function () { - return new Promise(function (resolve, reject) { +var renewJWTLogin = function() { + return new Promise(function(resolve, reject) { var duration = moment.duration(moment().diff(jwt_login_time)).asSeconds(); // if less than 60 seconds left before token expiry. if (duration > jwt_expiry - 60) { console.log(' - Requesting new ACRFD login token.'); - return loginToACRFD(username, password).then(function () { + return loginToACRFD(username, password).then(function() { resolve(); }); } else { @@ -178,94 +194,91 @@ var renewJWTLogin = function () { /** * Updates and saves the application features. - * - * - * @param {String} accessToken Tantalis api token - * @param {Application} oldApp application as it exists in ACRFD - * @param {Application} newApp application with the latest values from Tantalis - * @returns {Promise} + * @param {Application} acrfdApp application as it exists in ACRFD + * @param {Application} tantalisApp application with the latest values from Tantalis + * @returns {Promise} promise that resolves wih the updated ACRFD application */ -var getAndSaveFeatures = function (accessToken, item) { - return new Promise(function (resolve, reject) { - Utils.getApplicationByDispositionID(accessToken, item.tantalisID) - .then(function (obj) { - // console.log("returning:", obj); - // Store the features in the DB - var allFeaturesForDisp = []; - item.areaHectares = obj.areaHectares; +var updateFeatures = function(acrfdApp, tantalisApp) { + return new Promise(function(resolve, reject) { + // console.log("returning:", tantalisApp); + // Store the features in the DB + var allFeaturesForDisp = []; + acrfdApp.areaHectares = tantalisApp.areaHectares; - var turf = require('@turf/turf'); - var helpers = require('@turf/helpers'); - var centroids = helpers.featureCollection([]); - _.each(obj.parcels, function (f) { - // Tags default public - f.tags = [['sysadmin'], ['public']]; - // copy in all the app meta just to stay consistent. - f.properties.RESPONSIBLE_BUSINESS_UNIT = obj.RESPONSIBLE_BUSINESS_UNIT; - f.properties.TENURE_PURPOSE = obj.TENURE_PURPOSE; - f.properties.TENURE_SUBPURPOSE = obj.TENURE_SUBPURPOSE; - f.properties.TENURE_STATUS = obj.TENURE_STATUS; - f.properties.TENURE_TYPE = obj.TENURE_TYPE; - f.properties.TENURE_STAGE = obj.TENURE_STAGE; - f.properties.TENURE_SUBTYPE = obj.TENURE_SUBTYPE; - f.properties.TENURE_LOCATION = obj.TENURE_LOCATION; - f.properties.DISPOSITION_TRANSACTION_SID = obj.DISPOSITION_TRANSACTION_SID; - f.properties.CROWN_LANDS_FILE = obj.CROWN_LANDS_FILE; + var turf = require('@turf/turf'); + var helpers = require('@turf/helpers'); + var centroids = helpers.featureCollection([]); + _.each(tantalisApp.parcels, function(f) { + // Tags default public + f.tags = [['sysadmin'], ['public']]; + // copy in all the app meta just to stay consistent. + f.properties.RESPONSIBLE_BUSINESS_UNIT = tantalisApp.RESPONSIBLE_BUSINESS_UNIT; + f.properties.TENURE_PURPOSE = tantalisApp.TENURE_PURPOSE; + f.properties.TENURE_SUBPURPOSE = tantalisApp.TENURE_SUBPURPOSE; + f.properties.TENURE_STATUS = tantalisApp.TENURE_STATUS; + f.properties.TENURE_TYPE = tantalisApp.TENURE_TYPE; + f.properties.TENURE_STAGE = tantalisApp.TENURE_STAGE; + f.properties.TENURE_SUBTYPE = tantalisApp.TENURE_SUBTYPE; + f.properties.TENURE_LOCATION = tantalisApp.TENURE_LOCATION; + f.properties.DISPOSITION_TRANSACTION_SID = tantalisApp.DISPOSITION_TRANSACTION_SID; + f.properties.CROWN_LANDS_FILE = tantalisApp.CROWN_LANDS_FILE; - allFeaturesForDisp.push(f); - // Get the polygon and put it for later centroid calculation - centroids.features.push(turf.centroid(f)); - }); - // Centroid of all the shapes. - if (centroids.features.length > 0) { - item.centroid = turf.centroid(centroids).geometry.coordinates; - } - item.client = ""; - for (let [idx, client] of Object.entries(obj.interestedParties)) { - if (idx > 0) { - item.client += ", "; - } - if (client.interestedPartyType == 'O') { - item.client += client.legalName; - } else { - item.client += client.firstName + " " + client.lastName; - } - } - item.statusHistoryEffectiveDate = obj.statusHistoryEffectiveDate; + allFeaturesForDisp.push(f); + // Get the polygon and put it for later centroid calculation + centroids.features.push(turf.centroid(f)); + }); + // Centroid of all the shapes. + if (centroids.features.length > 0) { + acrfdApp.centroid = turf.centroid(centroids).geometry.coordinates; + } + acrfdApp.client = ''; + for (let [idx, client] of Object.entries(tantalisApp.interestedParties)) { + if (idx > 0) { + acrfdApp.client += ', '; + } + if (client.interestedPartyType == 'O') { + acrfdApp.client += client.legalName; + } else { + acrfdApp.client += client.firstName + ' ' + client.lastName; + } + } + acrfdApp.statusHistoryEffectiveDate = tantalisApp.statusHistoryEffectiveDate; - Promise.resolve() - .then(function () { - return allFeaturesForDisp.reduce(function (previousItem, currentItem) { - return previousItem.then(function () { - return doFeatureSave(currentItem, item._id); - }); - }, Promise.resolve()); - }).then(function () { - resolve({ app: item, tantalisApp: obj }); + Promise.resolve() + .then(function() { + return allFeaturesForDisp.reduce(function(previousFeature, currentFeature) { + return previousFeature.then(function() { + return saveFeatures(currentFeature, acrfdApp._id); }); + }, Promise.resolve()); + }) + .then(function() { + resolve(acrfdApp); }); }); }; /** - * Updates and saves the application features. + * Saves the application features. * - * @param {Application} item Application - * @param {String} appId Application id + * @param {Application} feature Application feature + * @param {String} acrfdAppId Application id * @returns {Promise} */ -var doFeatureSave = function (item, appId) { - return new Promise(function (resolve, reject) { - item.applicationID = appId; - request.post({ +var saveFeatures = function(feature, acrfdAppId) { + return new Promise(function(resolve, reject) { + feature.applicationID = acrfdAppId; + request.post( + { url: uri + 'api/feature', headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + jwt_login }, - body: JSON.stringify(item) + body: JSON.stringify(feature) }, - function (err, res, body) { + function(err, res, body) { if (err || res.statusCode !== 200) { console.log(' - doFeatureSave err:', err, res); reject(null); @@ -281,72 +294,115 @@ var doFeatureSave = function (item, appId) { /** * Updates and saves the ACRFD application meta. * - * @param {Application} acrfdItem - * @param {Object} tantalisItem + * @param {Application} acrfdApp + * @param {Object} tantalisApp * @returns */ -var updateApplicationMeta = function (acrfdItem, tantalisItem) { - return new Promise(function (resolve, reject) { +var updateApplicationMeta = function(acrfdApp, tantalisApp) { + return new Promise(function(resolve, reject) { var updatedAppObject = {}; - updatedAppObject.businessUnit = tantalisItem.RESPONSIBLE_BUSINESS_UNIT; - updatedAppObject.purpose = tantalisItem.TENURE_PURPOSE; - updatedAppObject.subpurpose = tantalisItem.TENURE_SUBPURPOSE; - updatedAppObject.status = tantalisItem.TENURE_STATUS; - updatedAppObject.type = tantalisItem.TENURE_TYPE; - updatedAppObject.tenureStage = tantalisItem.TENURE_STAGE; - updatedAppObject.subtype = tantalisItem.TENURE_SUBTYPE; - updatedAppObject.location = tantalisItem.TENURE_LOCATION; - updatedAppObject.legalDescription = tantalisItem.TENURE_LEGAL_DESCRIPTION; - updatedAppObject.centroid = acrfdItem.centroid; - updatedAppObject.areaHectares = acrfdItem.areaHectares; - updatedAppObject.client = acrfdItem.client; - updatedAppObject.statusHistoryEffectiveDate = acrfdItem.statusHistoryEffectiveDate; + updatedAppObject.businessUnit = tantalisApp.RESPONSIBLE_BUSINESS_UNIT; + updatedAppObject.purpose = tantalisApp.TENURE_PURPOSE; + updatedAppObject.subpurpose = tantalisApp.TENURE_SUBPURPOSE; + updatedAppObject.status = tantalisApp.TENURE_STATUS; + updatedAppObject.type = tantalisApp.TENURE_TYPE; + updatedAppObject.tenureStage = tantalisApp.TENURE_STAGE; + updatedAppObject.subtype = tantalisApp.TENURE_SUBTYPE; + updatedAppObject.location = tantalisApp.TENURE_LOCATION; + updatedAppObject.legalDescription = tantalisApp.TENURE_LEGAL_DESCRIPTION; + updatedAppObject.centroid = acrfdApp.centroid; + updatedAppObject.areaHectares = acrfdApp.areaHectares; + updatedAppObject.client = acrfdApp.client; + updatedAppObject.statusHistoryEffectiveDate = acrfdApp.statusHistoryEffectiveDate; - request.put({ - url: uri + 'api/application/' + acrfdItem._id, - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + jwt_login + request.put( + { + url: uri + 'api/application/' + acrfdApp._id, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + }, + body: JSON.stringify(updatedAppObject) }, - body: JSON.stringify(updatedAppObject), - }, function (err, res, body) { - if (err || res.statusCode !== 200) { - console.log(" - updateApplicationMeta err:", err, res); - reject(null); - } else { - var data = JSON.parse(body); - resolve(data); + function(err, res, body) { + if (err || res.statusCode !== 200) { + console.log(' - updateApplicationMeta err:', err, res); + reject(null); + } else { + var data = JSON.parse(body); + resolve(data); + } } - }); + ); }); }; -var retiredStatuses = ['ABANDONED', 'CANCELLED', 'OFFER NOT ACCEPTED', 'OFFER RESCINDED', 'RETURNED', 'REVERTED', 'SOLD', - 'SUSPENDED', 'WITHDRAWN', 'ACTIVE', 'COMPLETED', 'DISPOSITION IN GOOD STANDING', 'EXPIRED', 'HISTORIC', 'DISALLOWED']; +/** + * Given an ACRFD applications tantalisID (disposition ID), makes all necessary calls to update it with the latest information from Tantalis. + * + * @param {string} applicationIDToUpdate a tantalisID + * @returns {Promise} + */ +var updateApplication = function (applicationIDToUpdate) { + return renewJWTLogin() + .then(function() { + return getApplicationByID('api/application', applicationIDToUpdate); + }) + .then(function(applicationsToUpdate) { + // Only expecting 1 result, but the API returns an array + return applicationsToUpdate.reduce(function(previousApp, currentApp) { + return previousApp.then(function () { + console.log('-----------------------------------------------'); + console.log(`6. Updating ACRFD Application, tantalisID: ${currentApp.tantalisID}`); + console.log(' - Fetching Tantalis application'); + return Utils.getApplicationByDispositionID(_accessToken, currentApp.tantalisID) + .then(function(tantalisApp) { + if (!tantalisApp) { + console.log(' - No Tantalis application found - not updating.'); + return Promise.resolve(); + } + console.log(' - Deleting existing application features'); + return deleteAllApplicationFeatures(currentApp) + .then(function() { + console.log(' - Updating new application features'); + return updateFeatures(currentApp, tantalisApp) + }) + .then(function(updatedApp) { + console.log(' - Updating new application meta'); + return updateApplicationMeta(updatedApp, tantalisApp); + }) + }); + }); + }, Promise.resolve()); + }); +}; /** - * Fetches all ACRFD applications that have a retired status AND a statusHistoryEffectiveDate within the last week 6 months ago. + * Fetches all ACRFD applications that have a retired status AND a statusHistoryEffectiveDate within the past week 6 months ago. * * @returns {Promise} promise that resolves with the list of retired applications. */ -var getApplicationsToUnpublish = function () { - console.log(' - fetching retired applications.') - return new Promise(function (resolve, reject) { - var sinceDate = moment().subtract(6, 'months').subtract(1, 'week'); +var getApplicationsToUnpublish = function() { + console.log(' - fetching retired applications.'); + return new Promise(function(resolve, reject) { + var sinceDate = moment() + .subtract(6, 'months') + .subtract(1, 'week'); var untilDate = moment().subtract(6, 'months'); // get all applications that are in a retired status, and that have a last status update date within in the past week 6 months ago. - var queryString = `?statusHistoryEffectiveDate[since]=${sinceDate.toISOString()}&statusHistoryEffectiveDate[until]=${untilDate.toISOString()}` - retiredStatuses.forEach((status) => queryString += `&status[eq]=${encodeURIComponent(status)}`); + var queryString = `?statusHistoryEffectiveDate[since]=${sinceDate.toISOString()}&statusHistoryEffectiveDate[until]=${untilDate.toISOString()}`; + retiredStatuses.forEach(status => (queryString += `&status[eq]=${encodeURIComponent(status)}`)); - request.get({ - url: uri + 'api/application' + queryString, - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + jwt_login - } - }, - function (err, res, body) { + request.get( + { + url: uri + 'api/application' + queryString, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + } + }, + function(err, res, body) { if (err || res.statusCode !== 200) { console.log(' - getApplicationsToUnpublish err:', err, res); reject(null); @@ -362,7 +418,7 @@ var getApplicationsToUnpublish = function () { } ); }); -} +}; /** * Unpublishes ACRFD applications. @@ -370,20 +426,20 @@ var getApplicationsToUnpublish = function () { * @param {*} applicationsToUnpublish array of applications * @returns {Promise} */ -var unpublishApplications = function (applicationsToUnpublish) { - console.log(` - found ${applicationsToUnpublish.length} retired applications.`) - return applicationsToUnpublish.reduce(function (previousApp, currentApp) { - return previousApp.then(function () { - return new Promise(function (resolve, reject) { - request.put({ - url: uri + 'api/application/' + currentApp._id + '/unpublish', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + jwt_login +var unpublishApplications = function(applicationsToUnpublish) { + return applicationsToUnpublish.reduce(function(previousApp, currentApp) { + return previousApp.then(function() { + return new Promise(function(resolve, reject) { + request.put( + { + url: uri + 'api/application/' + currentApp._id + '/unpublish', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + }, + body: JSON.stringify(currentApp) }, - body: JSON.stringify(currentApp) - }, - function (err, res, body) { + function(err, res, body) { if (err || res.statusCode !== 200) { console.log(' - unpublishApplications err:', err, body); reject(null); @@ -393,89 +449,105 @@ var unpublishApplications = function (applicationsToUnpublish) { resolve(data); } } - ) + ); }); }); }, Promise.resolve()); }; +/** + * Gets all non-deleted ACRFD application tantalis IDs. + * + * @returns {Promise} promise that resolves with an array of ACRFD application tantalisIDs. + */ +var getAllApplicationIDs = function() { + return new Promise(function(resolve, reject) { + // only update the ones that aren't deleted + const url = uri + 'api/application/' + '?fields=tantalisID&isDeleted=false'; + request.get( + { + url: url, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + jwt_login + } + }, + function(err, res, body) { + if (err) { + console.log(' - getAllApplicationIDs err:', err); + reject(err); + } else if (res.statusCode !== 200) { + console.log('res.statusCode:', res.statusCode); + reject(res.statusCode + ' ' + body); + } else { + var obj = {}; + try { + obj = JSON.parse(body); + resolve(obj); + } catch (e) { + console.log(' - getAllApplicationIDs parse err:', e); + } + } + } + ); + }); +}; + /** * Main call chain that utilizes the above functions to update ACRFD applications. */ -console.log('======================================================='); console.log('1. Authenticating with ACRFD.'); loginToACRFD(username, password) - .then(function () { + .then(function() { console.log('-----------------------------------------------'); - console.log('2. Unpublishing retired applications.') - return getApplicationsToUnpublish() - .then(function (appsToUnpublish) { - return unpublishApplications(appsToUnpublish); - }); + console.log('2. Unpublishing retired applications.'); + return getApplicationsToUnpublish().then(function(applicationsToUnpublish) { + console.log(` - found ${applicationsToUnpublish.length} retired applications.`); + return unpublishApplications(applicationsToUnpublish); + }); }) - .then(function () { + .then(function() { console.log('-----------------------------------------------'); console.log('3. Authenticating with Tantalis.'); - return Utils.loginWebADE() - .then(function (accessToken) { - console.log(' - TTLS API login token:', accessToken); - _accessToken = accessToken; - return _accessToken; - }); + return Utils.loginWebADE().then(function(accessToken) { + console.log(' - TTLS API login token:', accessToken); + _accessToken = accessToken; + return _accessToken; + }); }) - .then(function () { + .then(function() { console.log('-----------------------------------------------'); console.log('4. Fetching all Tantalis applications that have been updated in the last day.'); - var lastDay = moment().subtract(1, 'days').format('YYYYMMDD'); + var lastDay = moment() + .subtract(1, 'days') + .format('YYYYMMDD'); return Utils.getAllApplicationIDs(_accessToken, { updated: lastDay }); }) - .then(function (recentlyUpdatedApplicationIDs) { - console.log(` - Found ${recentlyUpdatedApplicationIDs.length} recently updated Tantalis Applications.`); - // For each recently updated application from Tantalis, fetch the matching record in ACRFD, if it exists, and update it - return recentlyUpdatedApplicationIDs.reduce(function (previousItem, currentItem) { - return previousItem.then(function () { + .then(function(recentlyUpdatedApplicationIDs) { + console.log('-----------------------------------------------'); + console.log('5. Fetching all non-deleted ACRFD applications and cross referencing with recently updated Tantalis applications.'); + return getAllApplicationIDs().then(function(allACRFDApplicationIDs) { + return allACRFDApplicationIDs.map(app => app.tantalisID).filter(tantalisID => recentlyUpdatedApplicationIDs.includes(tantalisID)); + }); + }) + .then(function(applicationIDsToUpdate) { + console.log(` - Found ${applicationIDsToUpdate.length} ACRFD Applications with matching recently updated Tantalis application.`); + // For each ACRFD application with a matching recently updated application from Tantalis, fetch the matching record in ACRFD and update it + return applicationIDsToUpdate.reduce(function(previousItem, currentItem) { + return previousItem.then(function() { // Each iteration, check if the ACRFD login token needs to be re-fetched - return renewJWTLogin() - .then(function () { - console.log('-----------------------------------------------'); - console.log(`5. Attempting to find matching ACRFD Application, tantalisID: ${currentItem}`); - return getApplicationByID('api/application', currentItem); - }) - .then(function (matchingACRFDApplications) { - // Only expecting 1 result, but the API returns an array - return matchingACRFDApplications.reduce(function (previousApp, currentApp) { - return previousApp.then(function () { - console.log('-----------------------------------------------'); - console.log(`6. Matching ACRFD Application found`); - console.log(" - Deleting existing application features"); - return deleteAllApplicationFeatures(currentApp) - .then(function () { - console.log(" - Updating new application features"); - return getAndSaveFeatures(_accessToken, currentApp); - }) - .then(function ({ app, tantalisApp }) { - if (app && tantalisApp) { - console.log(" - Updating new application meta"); - return updateApplicationMeta(app, tantalisApp); - } else { - console.log(" - No features found - not updating."); - return Promise.resolve(); - } - }) - }); - }, Promise.resolve()); - }); + return updateApplication(currentItem); }); }, Promise.resolve()); }) - .then(function () { + .then(function() { console.log('-----------------------------------------------'); console.log('Done!'); console.log('======================================================='); }) - .catch(function (err) { + .catch(function(err) { console.log('-----------------------------------------------'); console.log(' - General err:', err); console.log('======================================================='); process.exit(1); - }); \ No newline at end of file + });