Merge pull request #105 from mpayson/mp-batch-geocode

new Node.js demo for batch geocoding a .csv

demos/batch-geocoder-node/.gitignore

@@ -0,0 +1,72 @@
+# Local
+config.js
+*.csv
+
+# vscode
+.vscode/
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# next.js build output
+.next

demos/batch-geocoder-node/README.md

@@ -0,0 +1,12 @@
+# Running this demo
+
+:loudspeaker: Node.js 8.6 or higher is [required](http://node.green/), this demo uses a couple shiny new JS features like `{... }`
+
+1. Make sure you run `npm run bootstrap` in the root folder to setup the dependencies
+2. Replace values in [config-template.js](/demos/batch-geocoder/config-template.js) and rename to `config.js`
+   1. `"un"`: ArcGIS username
+   2. `"pw"`: ArcGIS password
+   3. `"csv"`: csv path
+   4. `"output"`: output csv path
+   5. `"fieldmap"`: `object` that maps CSV fields to [address fields](https://esri.github.io/arcgis-rest-js/api/geocoder/IAddressBulk/) __or__ `string` that points to a CSV field with single-line addresses
+3. `node batch-geocode.js`

demos/batch-geocoder-node/batch-geocode.js

@@ -0,0 +1,109 @@
+require('isomorphic-fetch');
+require('isomorphic-form-data');
+const fs = require('fs');
+const Papa = require('papaparse');
+const { UserSession } = require('@esri/arcgis-rest-auth');
+const { bulkGeocode } = require('@esri/arcgis-rest-geocoder');
+const config = require('./config');
+
+// FUNCTIONS!
+
+// Reads a csv file to an array of dictionary rows
+const parseCsv = csvPath => {
+  const readStream = fs.createReadStream(csvPath);
+  return new Promise((res, rej) => {
+    Papa.parse(readStream, {
+      header: true,
+      complete: (data, file) => res(data.data),
+      error: (er, file) => rej(er)
+    });
+  });
+};
+
+//Writes a csv-style object to filepath
+const exportCsv = (output, filePath) => {
+  return new Promise((res, rej) => {
+    const str = Papa.unparse(output);
+    fs.writeFile(filePath, str, er => {
+      if(er) rej(er);
+      res("SUCCESS!");
+    });
+  });
+}
+
+// Format objects in data to conform to request params
+// `fields` can be a `string` for address field or an `object` mapping request fields to csv fields
+// https://esri.github.io/arcgis-rest-js/api/geocoder/IAddressBulk/
+const getAddresses = (data, fields) => {
+  if(typeof fields === 'string'){
+    return data.map((row, i) => ({
+      OBJECTID: i,
+      address: row[fields]
+    }));
+  }
+  return data.map((row,i) => {
+    let addressObj = {OBJECTID: i};
+    for(let key in fields){
+      addressObj[key] = row[fields[key]];
+    }
+    return addressObj;
+  });
+};
+
+// Chunks an array to max batch geocode limit of 1000
+// Copied from https://github.com/Chalarangelo/30-seconds-of-code#chunk
+const chunkGeocode = arr =>
+  Array.from({ length: Math.ceil(arr.length / 1000) }, (v, i) =>
+    arr.slice(i * 1000, i * 1000 + 1000)
+  );
+
+// Translates array of batch geocode request results to IDs mapped to geocode locations
+const mapResults = results =>
+  results.reduce((resMap, res) => {
+    const locations = res.locations;
+    return locations.reduce((locMap, loc) => {
+      const locFields = {};
+      locFields['x'] = loc.location.x;
+      locFields['y'] = loc.location.y;
+      locMap[loc.attributes.ResultID] = locFields;
+      return locMap;
+    }, resMap);
+  }, {});
+
+// IMPLEMENTATION!
+
+// Instantiate a user session to run Geocoding service
+const session = new UserSession({
+  username: config.un,
+  password: config.pw
+});
+
+// Parse and geocode
+parseCsv(config.csv)
+  .then(data => {
+    // Build address requests
+    const addrs = getAddresses(data, config.fieldmap);
+    const chunks = chunkGeocode(addrs, 1000);
+
+    // Geocode
+    const promises = chunks.map(chunk => 
+      bulkGeocode({addresses: chunk, authentication: session})
+    );
+
+    // Resolve results and combine with CSV data
+    return Promise.all(promises).then(res => {
+      resultMap = mapResults(res);
+      output = data.map((row, i) => {
+        id = row[config.fieldmap.OBJECTID] || i;
+        return {...row, ...resultMap[id]};
+      });
+      return output;
+    });
+  })
+  .then(output => {
+    // Write the new CSV
+    return exportCsv(output, config.output);
+  })
+  .then(success => console.log(success))
+  .catch(er => console.log(er))
+

demos/batch-geocoder-node/config-template.js

@@ -0,0 +1,18 @@
+module.exports = {
+  "un": "<USERNAME>",
+  "pw": "<PASSWORD>",
+  "csv": "NYC Restaurant Inspections.csv",
+  "output": "NYC Inspections Geocoded.csv",
+  "fieldmap": {
+    "address": "ADDRESS",
+    // "address2": "<csv field>",
+    // "address3": "<csv field>",
+    "city": "CITY",
+    // "countryCode": "<csv field>",
+    // "neighborhood": "<csv field>",
+    // "postal": "<csv field>",
+    // "postal Ext": "<csv field>",
+    "region": "STATE",
+    // "subregion": "<csv field>"
+  }
+}

demos/batch-geocoder-node/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "batch-geocoder",
+  "version": "1.0.2",
+  "description": "arcgis-rest-js batch geocode sample",
+  "main": "batch-geocode.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Esri/arcgis-rest-js.git"
+  },
+  "keywords": [
+    "geocode",
+    "batch",
+    "arcgis",
+    "rest",
+    "js",
+    "arcgis-rest-js"
+  ],
+  "author": "Max Payson (mpayson)",
+  "license": "Apache-2.0",
+  "private": true,
+  "bugs": {
+    "url": "https://github.com/Esri/arcgis-rest-js/issues"
+  },
+  "homepage": "https://github.com/Esri/arcgis-rest-js#readme",
+  "dependencies": {
+    "@esri/arcgis-rest-auth": "^1.0.2",
+    "@esri/arcgis-rest-common-types": "^1.0.2",
+    "@esri/arcgis-rest-geocoder": "^1.0.2",
+    "@esri/arcgis-rest-request": "^1.0.2",
+    "isomorphic-fetch": "^2.2.1",
+    "isomorphic-form-data": "^1.0.0",
+    "papaparse": "^4.3.6"
+  }
+}