From 74dde5030c271ac17a7388c05b13fcf6e9fa9cde Mon Sep 17 00:00:00 2001
From: Cindy Qi Li <cli@ocadu.ca>
Date: Wed, 27 Jun 2018 15:30:28 -0400
Subject: [PATCH 01/41] GPII-3138: Added CouchDB views for retrieving snapset
 data.

---
 scripts/convertPrefs.js    | 4 ++--
 testData/dbData/views.json | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/convertPrefs.js b/scripts/convertPrefs.js
index 5ec9cded8..3aa60a2e9 100644
--- a/scripts/convertPrefs.js
+++ b/scripts/convertPrefs.js
@@ -9,7 +9,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 */
 
 // This script reads files from an input directory that contains preferences JSON5 files and convert them to JSON files of GPII keys and
-// preferences safes suitable for direct loading into CouchDB or PouchDB, which comply with the new GPII data model: 
+// preferences safes suitable for direct loading into CouchDB or PouchDB, which comply with the new GPII data model:
 // https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences in the target directory
 // Usage: node scripts/convertPrefs.js {input_path} {target_path}
 //
@@ -61,7 +61,7 @@ rimraf(targetDir, function () {
                     "_id": prefsSafeId,
                     "type": "prefsSafe",
                     "schemaVersion": "0.1",
-                    "prefsSafeType": "user",
+                    "prefsSafeType": "snapset",
                     "name": gpiiKey,
                     "password": null,
                     "email": null,
diff --git a/testData/dbData/views.json b/testData/dbData/views.json
index 9c538651e..fcbc8d55e 100644
--- a/testData/dbData/views.json
+++ b/testData/dbData/views.json
@@ -10,6 +10,12 @@
             },
             "findAuthorizationByAccessToken": {
                 "map": "function(doc) {if (doc.type === 'gpiiAppInstallationAuthorization' && doc.revoked === false) {emit(doc.accessToken, {'_id': doc.clientId, 'authorization': doc})}}"
+            },
+            "findSnapsetPrefsSafes": {
+                "map": "function(doc) {if (doc.type === 'prefsSafe' && doc.prefsSafeType === 'snapset') {emit(doc._id, doc)}}"
+            },
+            "findGpiiKeysByPrefsSafeId": {
+                "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc.prefsSafeId, doc)}}"
             }
         }
     }

From ee30bb29819d79c70373e37f165924fdf8c10aba Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Fri, 6 Jul 2018 14:47:05 -0400
Subject: [PATCH 02/41] GPII-3138:  Update snapsets in the data base

Modified vagrantCloudBasedContainers.sh script to make use of
the changes in the GPII-3138 branch of gpii/gpii-dataloader
---
 scripts/vagrantCloudBasedContainers.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index b728a40e3..249da7748 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -32,9 +32,15 @@ COUCHDB_IMAGE=couchdb
 COUCHDB_PORT=5984
 COUCHDB_HEALTHCHECK_DELAY=2
 COUCHDB_HEALTHCHECK_TIMEOUT=30
+if [ "$NO_REBUILD" == "true" ] ; then
+    CLEAR_INDEX=1
+else
+    CLEAR_INDEX=
+fi
 
-STATIC_DATA_DIR="/home/vagrant/sync/universal/testData/dbData"
-BUILD_DATA_DIR="/home/vagrant/sync/universal/build/dbData"
+UNIVERSAL_DIR="/home/vagrant/sync/universal"
+STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
+BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData"
 
 DATALOADER_IMAGE="gpii/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
@@ -77,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=1 $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAD_INDEX -e NODE_PATH=$UNIVERSAL_DIR $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From 1c0f38a3824cefdb210cf89123b3b9dbf80e78b3 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Fri, 6 Jul 2018 14:53:36 -0400
Subject: [PATCH 03/41] GPII-3138:  Update snapsets in the data base

Moved deleteSnapset.js from gpii-dataloader to universal's
script folder.
---
 scripts/deleteSnapsets.js | 158 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100644 scripts/deleteSnapsets.js

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
new file mode 100644
index 000000000..7fac5eb18
--- /dev/null
+++ b/scripts/deleteSnapsets.js
@@ -0,0 +1,158 @@
+/*!
+Copyright 2018 OCAD University
+
+Licensed under the New BSD license. You may not use this file except in
+compliance with this License.
+
+You may obtain a copy of the License at
+https://github.com/GPII/universal/blob/master/LICENSE.txt
+*/
+
+// This script modifies the preferences data base:
+// 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"),
+// 2. Finds all the GPII Keys associated with each snapset Prefs Safe
+// 3. Deletes the found Prefs Safes and associated GPII Keys
+//
+// A sample command that runs this script:
+// node deleteAndLoadSnapsets.js $COUCHDBURL
+
+"use strict";
+
+var http = require('http'),
+    fluid = require("infusion");
+
+var gpii = fluid.registerNamespace("gpii");
+fluid.registerNamespace("gpii.dataLoader");
+fluid.setLogging(fluid.logLevel.INFO)
+
+var dbLoader = gpii.dataLoader;
+dbLoader.couchDBURL = process.argv[2];
+if (!fluid.isValue(dbLoader.couchDBURL)) {
+    fluid.log ("COUCHDB_URL environment variable must be defined");
+    process.exit(1);
+}
+fluid.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'");
+dbLoader.prefsSafesViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findSnapsetPrefsSafes";
+dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKeysByPrefsSafeId";
+dbLoader.docsToRemove=[];
+
+/**
+ * Find the Prefs Safes of type "snapSet", add them to an array of records to
+ * remove, then use them to find their associated GPII Key(s).
+ * @param {Object} response - The response from the data base containing the
+ *                            snapSet Prefs Safes as a JSON string.
+ */
+dbLoader.processSnapsets = function (response) {
+    var snapSetsString = "";
+    response.setEncoding("utf8");
+    response.on("data", function (chunk) {
+        snapSetsString += chunk;
+    });
+    response.on("end", function () {
+        dbLoader.snapSets = JSON.parse(snapSetsString);
+        dbLoader.snapSets.rows.forEach(function (aSnapset) {
+            aSnapset.value._deleted = true;
+            dbLoader.docsToRemove.push(aSnapset.value);
+        });
+        dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove);
+    });
+};
+
+/**
+ * Find the GPII key records associated with the given snapset Prefs Safes,
+ * delete each one as found, and then batch delete all the snapset Prefs Safes.
+ * @param {Array} snapSets      - The snapsets of interest.
+ * @param {Array} docsToRemove  - Array of records to delete.
+ */
+dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
+    fluid.each (snapSets, function (aSnapset) {
+        var gpiiKeyId = aSnapset.value._id;
+        var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22";
+        var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) {
+            var respString = "";
+            resp.setEncoding("utf8");
+            resp.on("data", function (chunk) {
+                respString += chunk;
+            });
+            // This response "end" event doesn't finish until after the call to
+            // doBatchDelete() below, which is too late to add the GPII Key to
+            // docsToRemove and have them removed.
+            // Note - if there is a way to determine the final "end" event, then
+            // it could  call doBatchDelete().
+            resp.on("end", function () {
+                var gpiiKeyRecords = JSON.parse(respString);
+                gpiiKeyRecords.rows.forEach(function (record) {
+                    dbLoader.deleteGpiiKey(record.value);
+                });
+            });
+        });
+        getGpiiKeysRequest.on("error", function (e) {
+            fluid.log("Error finding snapsets' associated GPII Keys: " + e.message);
+        });
+        getGpiiKeysRequest.end();
+    });
+    dbLoader.doBatchDelete(docsToRemove);
+};
+
+/**
+ * Delete the given GPII Key record.
+ * @param {Object} gpiiKey  - The key to delete.
+ */
+dbLoader.deleteGpiiKey = function (gpiiKey) {
+    var deleteOptions = {
+        hostname: "localhost",
+        port: 5984,
+        path: "",           // filled in below.
+        method: "DELETE",
+        headers: {
+            "Accept": "application/json"
+        }
+    };
+    deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev;
+    var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) {
+        fluid.log("STATUS: " + res.statusCode);
+        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
+    });
+    deleteGpiiKeyRequest.on("error", function (e) {
+        fluid.log("Error finding GPII Key: " + e.message);
+    });
+    deleteGpiiKeyRequest.end();
+};
+
+/**
+ * Delete the snapset Prefs Safes and their associated GPII Keys.
+ * @param {Array} docsToRemove  - Array of records to delete.
+ */
+dbLoader.doBatchDelete = function (docsToRemove) {
+    var batchDeleteOptions = {
+        hostname: "localhost",
+        port: 5984,
+        path: "/gpii/_bulk_docs",
+        method: "POST",
+        headers: {
+            "Accept": "application/json",
+            "Content-Length": 0,          // filled in below
+            "Content-Type": "application/json"
+        }
+    };
+    var batchPostData = JSON.stringify({"docs": docsToRemove});
+    batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+    var batchDeleteReq = http.request(batchDeleteOptions, function (res) {
+        fluid.log("STATUS: " + res.statusCode);
+        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
+        res.on('end', function () {
+            fluid.log('Batch deletion of snapsets');
+        });
+    });
+    batchDeleteReq.on('error', function (e) {
+        fluid.error("Error deleting snapset Prefs Safes: " + e.message);
+    });
+    batchDeleteReq.write(batchPostData);
+    batchDeleteReq.end();
+};
+
+dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets);
+dbLoader.snapSetsRequest.on("error", function (e) {
+    fluid.log("Error finding snapsets Prefs Safes: " + e.message);
+});
+dbLoader.snapSetsRequest.end();

From b5dd8c05063311bd35ce3bda3ff062f66ff21e5d Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Fri, 6 Jul 2018 16:56:55 -0400
Subject: [PATCH 04/41] GPII-3138:  Update snapsets in the data base

Fixed some (grievous) typos.
---
 scripts/vagrantCloudBasedContainers.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index 249da7748..ff37f9854 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -33,9 +33,9 @@ COUCHDB_PORT=5984
 COUCHDB_HEALTHCHECK_DELAY=2
 COUCHDB_HEALTHCHECK_TIMEOUT=30
 if [ "$NO_REBUILD" == "true" ] ; then
-    CLEAR_INDEX=1
-else
     CLEAR_INDEX=
+else
+    CLEAR_INDEX=1
 fi
 
 UNIVERSAL_DIR="/home/vagrant/sync/universal"
@@ -83,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAD_INDEX -e NODE_PATH=$UNIVERSAL_DIR $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX -e NODE_PATH=$UNIVERSAL_DIR $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From f7068c05130f3cf1573cbdf9c0e701bac5bbfd16 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 9 Jul 2018 15:05:59 -0400
Subject: [PATCH 05/41] GPII-3138:  Update snapsets in the data base

Fixed erroneous call to fluid.error() -- replaced with fluid.log().
---
 scripts/deleteSnapsets.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
index 7fac5eb18..b33d54836 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteSnapsets.js
@@ -145,7 +145,7 @@ dbLoader.doBatchDelete = function (docsToRemove) {
         });
     });
     batchDeleteReq.on('error', function (e) {
-        fluid.error("Error deleting snapset Prefs Safes: " + e.message);
+        fluid.log("Error deleting snapset Prefs Safes: " + e.message);
     });
     batchDeleteReq.write(batchPostData);
     batchDeleteReq.end();

From e2f4e63fc57493e617fe1ea6101094558abbfde7 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 9 Jul 2018 15:31:31 -0400
Subject: [PATCH 06/41] GPII-3138:  Update snapsets in the data base

Removed hard-coded host ("localhost") and port for the CouchDB
URL and used the actual value passed in on the command line.
---
 scripts/deleteSnapsets.js | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
index b33d54836..24f9e4efd 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteSnapsets.js
@@ -18,7 +18,8 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 
 "use strict";
 
-var http = require('http'),
+var http = require("http"),
+    url = require("url"),
     fluid = require("infusion");
 
 var gpii = fluid.registerNamespace("gpii");
@@ -26,14 +27,15 @@ fluid.registerNamespace("gpii.dataLoader");
 fluid.setLogging(fluid.logLevel.INFO)
 
 var dbLoader = gpii.dataLoader;
-dbLoader.couchDBURL = process.argv[2];
-if (!fluid.isValue(dbLoader.couchDBURL)) {
+dbLoader.couchDbUrl = process.argv[2];
+if (!fluid.isValue(dbLoader.couchDbUrl)) {
     fluid.log ("COUCHDB_URL environment variable must be defined");
     process.exit(1);
 }
-fluid.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'");
-dbLoader.prefsSafesViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findSnapsetPrefsSafes";
-dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKeysByPrefsSafeId";
+fluid.log("COUCHDB_URL: '" + dbLoader.couchDbUrl + "'");
+dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
+dbLoader.gpiiKeyViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findGpiiKeysByPrefsSafeId";
+dbLoader.parsedCouchDbUrl = url.parse(dbLoader.couchDbUrl);
 dbLoader.docsToRemove=[];
 
 /**
@@ -100,8 +102,8 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
  */
 dbLoader.deleteGpiiKey = function (gpiiKey) {
     var deleteOptions = {
-        hostname: "localhost",
-        port: 5984,
+        hostname: dbLoader.parsedCouchDbUrl.hostname,
+        port: dbLoader.parsedCouchDbUrl.port,
         path: "",           // filled in below.
         method: "DELETE",
         headers: {
@@ -125,8 +127,8 @@ dbLoader.deleteGpiiKey = function (gpiiKey) {
  */
 dbLoader.doBatchDelete = function (docsToRemove) {
     var batchDeleteOptions = {
-        hostname: "localhost",
-        port: 5984,
+        hostname: dbLoader.parsedCouchDbUrl.hostname,
+        port: dbLoader.parsedCouchDbUrl.port,
         path: "/gpii/_bulk_docs",
         method: "POST",
         headers: {

From 554ae823059f162dc2666692243493315e861e8a Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 9 Jul 2018 16:41:21 -0400
Subject: [PATCH 07/41] GPII-3138:  Update snapsets in the data base

- Replaced all occurrences of forEach() with fluid.each().
- Properly set up shell environment variable NODE_PATH.
---
 scripts/deleteSnapsets.js              | 6 +++---
 scripts/vagrantCloudBasedContainers.sh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
index 24f9e4efd..cae5b8f6c 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteSnapsets.js
@@ -52,7 +52,7 @@ dbLoader.processSnapsets = function (response) {
     });
     response.on("end", function () {
         dbLoader.snapSets = JSON.parse(snapSetsString);
-        dbLoader.snapSets.rows.forEach(function (aSnapset) {
+        fluid.each(dbLoader.snapSets.rows, function (aSnapset) {
             aSnapset.value._deleted = true;
             dbLoader.docsToRemove.push(aSnapset.value);
         });
@@ -67,7 +67,7 @@ dbLoader.processSnapsets = function (response) {
  * @param {Array} docsToRemove  - Array of records to delete.
  */
 dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
-    fluid.each (snapSets, function (aSnapset) {
+    fluid.each(snapSets, function (aSnapset) {
         var gpiiKeyId = aSnapset.value._id;
         var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22";
         var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) {
@@ -83,7 +83,7 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
             // it could  call doBatchDelete().
             resp.on("end", function () {
                 var gpiiKeyRecords = JSON.parse(respString);
-                gpiiKeyRecords.rows.forEach(function (record) {
+                fluid.each(gpiiKeyRecords.rows, function (record) {
                     dbLoader.deleteGpiiKey(record.value);
                 });
             });
diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index ff37f9854..6299c943b 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -83,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX -e NODE_PATH=$UNIVERSAL_DIR $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX -v $UNIVERSAL_DIR:/universal -e NODE_PATH=/universal $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From f145f661c262ebd7fef8c2f63cff7853acfd49dc Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 10 Jul 2018 14:18:09 -0400
Subject: [PATCH 08/41] GPII-3138:  Delete and reload snapsets in the data base

Added more log messages.
---
 scripts/deleteSnapsets.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
index cae5b8f6c..7edd529d0 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteSnapsets.js
@@ -14,7 +14,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 // 3. Deletes the found Prefs Safes and associated GPII Keys
 //
 // A sample command that runs this script:
-// node deleteAndLoadSnapsets.js $COUCHDBURL
+// node deleteSnapsets.js $COUCHDBURL
 
 "use strict";
 
@@ -69,6 +69,7 @@ dbLoader.processSnapsets = function (response) {
 dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
     fluid.each(snapSets, function (aSnapset) {
         var gpiiKeyId = aSnapset.value._id;
+        fluid.log("Snapset: " + gpiiKeyId);
         var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22";
         var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) {
             var respString = "";
@@ -83,8 +84,9 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
             // it could  call doBatchDelete().
             resp.on("end", function () {
                 var gpiiKeyRecords = JSON.parse(respString);
-                fluid.each(gpiiKeyRecords.rows, function (record) {
-                    dbLoader.deleteGpiiKey(record.value);
+                fluid.each(gpiiKeyRecords.rows, function (gpiiKey) {
+                    fluid.log("GPII Key: " + gpiiKey.value._id);
+                    dbLoader.deleteGpiiKey(gpiiKey.value);
                 });
             });
         });

From 67f7bd6b7149ecf3667995712849968389664028 Mon Sep 17 00:00:00 2001
From: Cindy Qi Li <cli@ocadu.ca>
Date: Wed, 18 Jul 2018 16:10:41 -0400
Subject: [PATCH 09/41] GPII-3138: Add a couchDB view to return all GPII keys.

---
 testData/dbData/views.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/testData/dbData/views.json b/testData/dbData/views.json
index fcbc8d55e..b6ab3dc0c 100644
--- a/testData/dbData/views.json
+++ b/testData/dbData/views.json
@@ -16,6 +16,9 @@
             },
             "findGpiiKeysByPrefsSafeId": {
                 "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc.prefsSafeId, doc)}}"
+            },
+            "findAllGpiiKeys": {
+                "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc._id, doc)}}"
             }
         }
     }

From 5f2f1b5ea75ade5dae0a429513070caa1c4c09bf Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Fri, 20 Jul 2018 11:15:34 -0400
Subject: [PATCH 10/41] GPII-3138:  Delete and (re)load snapsets into the
 database

- Refactored deleteSnapsets.js to use promises.
- Modified Dockerfile to remove dependency on NODE_ENV, as it is
not needed by the gpii-dataloader script built into the docker image.
---
 scripts/deleteSnapsets.js              | 289 ++++++++++++++++---------
 scripts/vagrantCloudBasedContainers.sh |   2 +-
 2 files changed, 186 insertions(+), 105 deletions(-)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteSnapsets.js
index 7edd529d0..ee57583dd 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteSnapsets.js
@@ -29,134 +29,215 @@ fluid.setLogging(fluid.logLevel.INFO)
 var dbLoader = gpii.dataLoader;
 dbLoader.couchDbUrl = process.argv[2];
 if (!fluid.isValue(dbLoader.couchDbUrl)) {
-    fluid.log ("COUCHDB_URL environment variable must be defined");
+    fluid.log("COUCHDB_URL environment variable must be defined");
+    fluid.log("Usage:  node deleteSnapsets.js $COUCHDB_URL");
     process.exit(1);
 }
-fluid.log("COUCHDB_URL: '" + dbLoader.couchDbUrl + "'");
 dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
-dbLoader.gpiiKeyViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findGpiiKeysByPrefsSafeId";
+dbLoader.gpiiKeysViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
 dbLoader.parsedCouchDbUrl = url.parse(dbLoader.couchDbUrl);
-dbLoader.docsToRemove=[];
+dbLoader.snapsetPrefsSafes = [];
+dbLoader.gpiiKeys = [];
+
+fluid.log("COUCHDB_URL: '" +
+    dbLoader.parsedCouchDbUrl.protocol + "//" +
+    dbLoader.parsedCouchDbUrl.hostname + ":" +
+    dbLoader.parsedCouchDbUrl.port +
+    dbLoader.parsedCouchDbUrl.pathname +
+"'");
 
 /**
- * Find the Prefs Safes of type "snapSet", add them to an array of records to
- * remove, then use them to find their associated GPII Key(s).
- * @param {Object} response - The response from the data base containing the
- *                            snapSet Prefs Safes as a JSON string.
+ * Find the Prefs Safes of type "snapset", mark them to be deleted and add
+ * them to an array of records to remove.
+ * @param {String} responseString - The response from the database query for
+ *                                  retrieving the snapset PrefsSafes records
+ * @return {Array} - Array of snapset Prefs Safes each with their "deleted"
+ *                   field set to true.
  */
-dbLoader.processSnapsets = function (response) {
-    var snapSetsString = "";
-    response.setEncoding("utf8");
-    response.on("data", function (chunk) {
-        snapSetsString += chunk;
-    });
-    response.on("end", function () {
-        dbLoader.snapSets = JSON.parse(snapSetsString);
-        fluid.each(dbLoader.snapSets.rows, function (aSnapset) {
-            aSnapset.value._deleted = true;
-            dbLoader.docsToRemove.push(aSnapset.value);
-        });
-        dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove);
+dbLoader.processSnapsets = function (responseString) {
+    fluid.log("Processing the snapset Prefs Safes records...");
+    dbLoader.snapSets = JSON.parse(responseString);
+    fluid.each(dbLoader.snapSets.rows, function (aSnapset) {
+        aSnapset.value._deleted = true;
+        dbLoader.snapsetPrefsSafes.push(aSnapset.value);
     });
+    fluid.log("\tSnapset Prefs Safes marked for deletion.");
+    return dbLoader.snapsetPrefsSafes;
 };
 
 /**
- * Find the GPII key records associated with the given snapset Prefs Safes,
- * delete each one as found, and then batch delete all the snapset Prefs Safes.
- * @param {Array} snapSets      - The snapsets of interest.
- * @param {Array} docsToRemove  - Array of records to delete.
+ * Find the GPII Key records that are associated with a snapset PrefsSafe, mark
+ * them for deletion, and add them to array of records to delete.
+ * @param {String} responseString - The response from the database query for
+ *                                  retrieving all the GPII Keys.
+ * @return {Array} - Array of snapset PrefsSafes' GPII Keys with the "deleted"
+ *                   field set to true.
  */
-dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
-    fluid.each(snapSets, function (aSnapset) {
-        var gpiiKeyId = aSnapset.value._id;
-        fluid.log("Snapset: " + gpiiKeyId);
-        var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22";
-        var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) {
-            var respString = "";
-            resp.setEncoding("utf8");
-            resp.on("data", function (chunk) {
-                respString += chunk;
-            });
-            // This response "end" event doesn't finish until after the call to
-            // doBatchDelete() below, which is too late to add the GPII Key to
-            // docsToRemove and have them removed.
-            // Note - if there is a way to determine the final "end" event, then
-            // it could  call doBatchDelete().
-            resp.on("end", function () {
-                var gpiiKeyRecords = JSON.parse(respString);
-                fluid.each(gpiiKeyRecords.rows, function (gpiiKey) {
-                    fluid.log("GPII Key: " + gpiiKey.value._id);
-                    dbLoader.deleteGpiiKey(gpiiKey.value);
-                });
-            });
-        });
-        getGpiiKeysRequest.on("error", function (e) {
-            fluid.log("Error finding snapsets' associated GPII Keys: " + e.message);
-        });
-        getGpiiKeysRequest.end();
-    });
-    dbLoader.doBatchDelete(docsToRemove);
+dbLoader.processGpiiKeys = function (responseString) {
+    fluid.log("Processing the GPII Keys...");
+    var gpiiKeyRecords = JSON.parse(responseString);
+    dbLoader.gpiiKeys = dbLoader.findAndDeletePrefsSafesGpiiKeys(
+        gpiiKeyRecords, dbLoader.snapsetPrefsSafes
+    );
+    fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
+    return dbLoader.gpiiKeys;
 };
 
 /**
- * Delete the given GPII Key record.
- * @param {Object} gpiiKey  - The key to delete.
+ * Given all the GPII Keys records in the database, find the ones that reference
+ * a snapset PrefsSafe.  As each GPII Key is found it is marked for
+ * deletion.
+ * @param {Array} gpiiKeyRecords - Array of GPII Key records from the database.
+ * @return {Array} - the values from the gpiiKeyRecords that are snapset GPII Keys.
  */
-dbLoader.deleteGpiiKey = function (gpiiKey) {
-    var deleteOptions = {
-        hostname: dbLoader.parsedCouchDbUrl.hostname,
-        port: dbLoader.parsedCouchDbUrl.port,
-        path: "",           // filled in below.
-        method: "DELETE",
-        headers: {
-            "Accept": "application/json"
+dbLoader.findAndDeletePrefsSafesGpiiKeys = function (gpiiKeyRecords, snapSets) {
+    var gpiiKeysToDelete = [];
+    fluid.each(gpiiKeyRecords.rows, function (gpiiKeyRecord) {
+        var gpiiKey = fluid.find(snapSets, function (aSnapSet) {
+            if (gpiiKeyRecord.value.prefsSafeId === aSnapSet._id) {
+                return gpiiKeyRecord.value;
+            }
+        }, null);
+        if (gpiiKey !== null) {
+            gpiiKey._deleted = true;
+            gpiiKeysToDelete.push(gpiiKey);
         }
-    };
-    deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev;
-    var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) {
-        fluid.log("STATUS: " + res.statusCode);
-        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
     });
-    deleteGpiiKeyRequest.on("error", function (e) {
-        fluid.log("Error finding GPII Key: " + e.message);
-    });
-    deleteGpiiKeyRequest.end();
+    return gpiiKeysToDelete;
 };
 
 /**
- * Delete the snapset Prefs Safes and their associated GPII Keys.
- * @param {Array} docsToRemove  - Array of records to delete.
+ * Log that the batch deletion of snapset Prefs Safes and their GPII Keys has
+ * been completed.
  */
-dbLoader.doBatchDelete = function (docsToRemove) {
-    var batchDeleteOptions = {
-        hostname: dbLoader.parsedCouchDbUrl.hostname,
-        port: dbLoader.parsedCouchDbUrl.port,
-        path: "/gpii/_bulk_docs",
-        method: "POST",
-        headers: {
-            "Accept": "application/json",
-            "Content-Length": 0,          // filled in below
-            "Content-Type": "application/json"
-        }
+dbLoader.procesBatchDelete = function () {
+    fluid.log("Bulk deletion of snapset Prefs Safes and their GPII Keys...");
+};
+
+/**
+ * Make a bulk request of the database to delete the snapset Prefs Safes and
+ * their associated GPII Keys in one go.  Note that this only sets up the request
+ * and returns a wrapper (function) to execute the request.
+ * @responseHandler {Object} - http response handler for the request.
+ * @return {Function} - A function that wraps an http request to execute the
+ *                      batch deletion.
+ */
+dbLoader.doBatchDelete = function (responseHandler) {
+    return function() {
+        var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
+
+        var batchDeleteOptions = {
+            hostname: dbLoader.parsedCouchDbUrl.hostname,
+            port: dbLoader.parsedCouchDbUrl.port,
+            path: "/gpii/_bulk_docs",
+            method: "POST",
+            headers: {
+                "Accept": "application/json",
+                "Content-Length": 0,          // filled in below
+                "Content-Type": "application/json"
+            }
+        };
+        var batchPostData = JSON.stringify({"docs": docsToRemove});
+        batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+        var batchDeleteRequest = http.request(batchDeleteOptions, responseHandler);
+        batchDeleteRequest.write(batchPostData);
+        batchDeleteRequest.end();
+        return batchDeleteRequest;
     };
-    var batchPostData = JSON.stringify({"docs": docsToRemove});
-    batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
-    var batchDeleteReq = http.request(batchDeleteOptions, function (res) {
-        fluid.log("STATUS: " + res.statusCode);
-        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
-        res.on('end', function () {
-            fluid.log('Batch deletion of snapsets');
+};
+
+/**
+ * Log that the snapsets (Prefs Safes and GPII Keys) have been deleted.
+ */
+dbLoader.bulkDeletionComplete = function() {
+    fluid.log ("\tBulk deletion completed.");
+};
+
+/**
+ * Generate a response handler, setting up the given promise to resolve/reject
+ * at the correct time.
+ * @handleEnd {Function} - Function to call that deals with the response data
+ *                         when the response receives an "end" event.
+ * @promise {Promise} - Promose to resolve/reject on a response "end" or "error"
+ *                      event.
+ * @errorMsg {String} - Optional error message to prepend to the error received
+ *                      from a response "error" event.
+ * @return {Function} - Function that acts as a reponse callback for an http
+ *                      request
+ */
+dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
+    return function (response) {
+        var responseString = "";
+
+        response.setEncoding("utf8");
+        response.on("data", function (chunk) {
+            responseString += chunk;
         });
+        response.on("end", function () {
+            debugger;
+            var value = handleEnd(responseString)
+            promise.resolve(value);
+            debugger;
+        });
+        response.on("error", function (e) {
+            fluid.log(errorMsg + e.message);
+            promise.reject(e);
+        });
+    };
+};
+
+/**
+ * General mechanism to create a database request, set up an error handler and
+ * return.  It is up to the caller to trigger the request by calling its end()
+ * function.
+ * @param {String} databaseURL - URL to query the database with.
+ * @param {Function} handleResponse - callback that processes the response from
+ *                                    the request.
+ * @param {String} errorMsg - optional error message for request errors.
+ * @return {http.ClientRequest} - The http request object.
+ */
+dbLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg) {
+    var aRequest = http.request(databaseURL, handleResponse);
+    aRequest.on("error", function (e) {
+        fluid.log(errorMsg + e.message);
     });
-    batchDeleteReq.on('error', function (e) {
-        fluid.log("Error deleting snapset Prefs Safes: " + e.message);
-    });
-    batchDeleteReq.write(batchPostData);
-    batchDeleteReq.end();
+    return aRequest;
 };
 
-dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets);
-dbLoader.snapSetsRequest.on("error", function (e) {
-    fluid.log("Error finding snapsets Prefs Safes: " + e.message);
-});
-dbLoader.snapSetsRequest.end();
+// Get the snapsets Prefs Safes.
+var snapsetsPromise = fluid.promise();
+var getSnapSetsResponse = dbLoader.createResponseHandler(
+    dbLoader.processSnapsets,
+    snapsetsPromise,
+    "Error retrieving snapsets Prefs Safes: "
+);
+var snapSetsRequest = dbLoader.queryDatabase(
+    dbLoader.prefsSafesViewUrl,
+    getSnapSetsResponse,
+    "Error requesting snapsets Prefs Safes: "
+);
+snapSetsRequest.end();
+
+// Get the associated GPII Keys.
+var gpiiKeysPromise = fluid.promise();
+var getGpiiKeysResponse = dbLoader.createResponseHandler(
+    dbLoader.processGpiiKeys,
+    gpiiKeysPromise,
+    "Error finding snapset Prefs Safes associated GPII Keys: "
+);
+var getGpiiKeysRequest = dbLoader.queryDatabase(
+    dbLoader.gpiiKeysViewUrl,
+    getGpiiKeysResponse,
+    "Error requesting GPII Keys: "
+);
+snapsetsPromise.then(function () { getGpiiKeysRequest.end(); });
+
+// Batch delete snapset Prefs Safes and their GPII Keys.
+var batchDeletePromise = fluid.promise();
+var batchDeleteResponse = dbLoader.createResponseHandler(dbLoader.procesBatchDelete, batchDeletePromise);
+var execBatchDelete = dbLoader.doBatchDelete(batchDeleteResponse);
+gpiiKeysPromise.then(execBatchDelete);
+
+// Done.
+batchDeletePromise.then(dbLoader.bulkDeletionComplete);
+
diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index 6299c943b..44c280138 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -83,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX -v $UNIVERSAL_DIR:/universal -e NODE_PATH=/universal $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From 750cdf3148743f85e2cdf3f43c64734462d4fba7 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 23 Jul 2018 10:37:24 -0400
Subject: [PATCH 11/41] GPII-3138:  Delete and reload snapsets into the
 database

- Added loading of the static and build data from disk (the
snapsets, gpii keys, and credentials).  This was previously done
in a bash script in gpii-dataloader (deleteAndLoadSnapsets.sh).
- Renamed the script to "deleteAndLoadSnapsets.js".
---
 ...teSnapsets.js => deleteAndLoadSnapsets.js} | 135 +++++++++++-------
 1 file changed, 85 insertions(+), 50 deletions(-)
 rename scripts/{deleteSnapsets.js => deleteAndLoadSnapsets.js} (62%)

diff --git a/scripts/deleteSnapsets.js b/scripts/deleteAndLoadSnapsets.js
similarity index 62%
rename from scripts/deleteSnapsets.js
rename to scripts/deleteAndLoadSnapsets.js
index ee57583dd..7c7f658ff 100644
--- a/scripts/deleteSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -20,19 +20,24 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 
 var http = require("http"),
     url = require("url"),
+    fs = require("fs"),
     fluid = require("infusion");
 
 var gpii = fluid.registerNamespace("gpii");
 fluid.registerNamespace("gpii.dataLoader");
-fluid.setLogging(fluid.logLevel.INFO)
+fluid.setLogging(fluid.logLevel.INFO);
 
 var dbLoader = gpii.dataLoader;
-dbLoader.couchDbUrl = process.argv[2];
-if (!fluid.isValue(dbLoader.couchDbUrl)) {
-    fluid.log("COUCHDB_URL environment variable must be defined");
-    fluid.log("Usage:  node deleteSnapsets.js $COUCHDB_URL");
+
+// Handle command line
+if (process.argv.length !== 5) {
+    fluid.log("Usage: node deleteSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR");
     process.exit(1);
 }
+dbLoader.couchDbUrl = process.argv[2];
+dbLoader.staticDataDir = process.argv[3];
+dbLoader.buildDataDir = process.argv[4];
+
 dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
 dbLoader.gpiiKeysViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
 dbLoader.parsedCouchDbUrl = url.parse(dbLoader.couchDbUrl);
@@ -46,13 +51,14 @@ fluid.log("COUCHDB_URL: '" +
     dbLoader.parsedCouchDbUrl.pathname +
 "'");
 
+fluid.log("STATIC_DATA_DIR: '" + dbLoader.staticDataDir + "'");
+fluid.log("BUILD_DATA_DIR: '" + dbLoader.buildDataDir + "'");
+
 /**
  * Find the Prefs Safes of type "snapset", mark them to be deleted and add
  * them to an array of records to remove.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving the snapset PrefsSafes records
- * @return {Array} - Array of snapset Prefs Safes each with their "deleted"
- *                   field set to true.
  */
 dbLoader.processSnapsets = function (responseString) {
     fluid.log("Processing the snapset Prefs Safes records...");
@@ -62,7 +68,6 @@ dbLoader.processSnapsets = function (responseString) {
         dbLoader.snapsetPrefsSafes.push(aSnapset.value);
     });
     fluid.log("\tSnapset Prefs Safes marked for deletion.");
-    return dbLoader.snapsetPrefsSafes;
 };
 
 /**
@@ -70,17 +75,14 @@ dbLoader.processSnapsets = function (responseString) {
  * them for deletion, and add them to array of records to delete.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving all the GPII Keys.
- * @return {Array} - Array of snapset PrefsSafes' GPII Keys with the "deleted"
- *                   field set to true.
  */
 dbLoader.processGpiiKeys = function (responseString) {
     fluid.log("Processing the GPII Keys...");
     var gpiiKeyRecords = JSON.parse(responseString);
-    dbLoader.gpiiKeys = dbLoader.findAndDeletePrefsSafesGpiiKeys(
+    dbLoader.gpiiKeys = dbLoader.markPrefsSafesGpiiKeysForDeletion(
         gpiiKeyRecords, dbLoader.snapsetPrefsSafes
     );
     fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
-    return dbLoader.gpiiKeys;
 };
 
 /**
@@ -88,9 +90,11 @@ dbLoader.processGpiiKeys = function (responseString) {
  * a snapset PrefsSafe.  As each GPII Key is found it is marked for
  * deletion.
  * @param {Array} gpiiKeyRecords - Array of GPII Key records from the database.
+ * @param {Array} snapSets - Array of snapset Prefs Safes whose id references
+ *                           its associated GPII Key record.
  * @return {Array} - the values from the gpiiKeyRecords that are snapset GPII Keys.
  */
-dbLoader.findAndDeletePrefsSafesGpiiKeys = function (gpiiKeyRecords, snapSets) {
+dbLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets) {
     var gpiiKeysToDelete = [];
     fluid.each(gpiiKeyRecords.rows, function (gpiiKeyRecord) {
         var gpiiKey = fluid.find(snapSets, function (aSnapSet) {
@@ -107,26 +111,15 @@ dbLoader.findAndDeletePrefsSafesGpiiKeys = function (gpiiKeyRecords, snapSets) {
 };
 
 /**
- * Log that the batch deletion of snapset Prefs Safes and their GPII Keys has
- * been completed.
- */
-dbLoader.procesBatchDelete = function () {
-    fluid.log("Bulk deletion of snapset Prefs Safes and their GPII Keys...");
-};
-
-/**
- * Make a bulk request of the database to delete the snapset Prefs Safes and
- * their associated GPII Keys in one go.  Note that this only sets up the request
- * and returns a wrapper (function) to execute the request.
- * @responseHandler {Object} - http response handler for the request.
+ * Create a function that makes a bulk docs POST request using the given data.
+ * @param {Object} dataToPost - JSON data to POST and process in bulk.
+ * @param {Object} responseHandler - http response handler for the request.
  * @return {Function} - A function that wraps an http request to execute the
- *                      batch deletion.
+ *                      POST.
  */
-dbLoader.doBatchDelete = function (responseHandler) {
-    return function() {
-        var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
-
-        var batchDeleteOptions = {
+dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler) {
+    return function () {
+        var postOptions = {
             hostname: dbLoader.parsedCouchDbUrl.hostname,
             port: dbLoader.parsedCouchDbUrl.port,
             path: "/gpii/_bulk_docs",
@@ -137,9 +130,9 @@ dbLoader.doBatchDelete = function (responseHandler) {
                 "Content-Type": "application/json"
             }
         };
-        var batchPostData = JSON.stringify({"docs": docsToRemove});
-        batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
-        var batchDeleteRequest = http.request(batchDeleteOptions, responseHandler);
+        var batchPostData = JSON.stringify({"docs": dataToPost});
+        postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+        var batchDeleteRequest = http.request(postOptions, responseHandler);
         batchDeleteRequest.write(batchPostData);
         batchDeleteRequest.end();
         return batchDeleteRequest;
@@ -149,21 +142,20 @@ dbLoader.doBatchDelete = function (responseHandler) {
 /**
  * Log that the snapsets (Prefs Safes and GPII Keys) have been deleted.
  */
-dbLoader.bulkDeletionComplete = function() {
+dbLoader.bulkDeletionComplete = function () {
     fluid.log ("\tBulk deletion completed.");
 };
 
 /**
  * Generate a response handler, setting up the given promise to resolve/reject
  * at the correct time.
- * @handleEnd {Function} - Function to call that deals with the response data
- *                         when the response receives an "end" event.
- * @promise {Promise} - Promose to resolve/reject on a response "end" or "error"
- *                      event.
- * @errorMsg {String} - Optional error message to prepend to the error received
- *                      from a response "error" event.
- * @return {Function} - Function that acts as a reponse callback for an http
- *                      request
+ * @param {Function} handleEnd - Function to call that deals with the response
+ *                               data when the response receives an "end" event.
+ * @param {Promise} promise - Promose to resolve/reject on a response "end" or
+ *                           "error" event.
+ * @param {String} errorMsg - Optional error message to prepend to the error
+ *                            received from a response "error" event.
+ * @return {Function} - Function reponse callback for an http request.
  */
 dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
     return function (response) {
@@ -174,10 +166,8 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
             responseString += chunk;
         });
         response.on("end", function () {
-            debugger;
-            var value = handleEnd(responseString)
+            var value = handleEnd(responseString);
             promise.resolve(value);
-            debugger;
         });
         response.on("error", function (e) {
             fluid.log(errorMsg + e.message);
@@ -204,6 +194,25 @@ dbLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg) {
     return aRequest;
 };
 
+/**
+ * Get all the json files from the given directory, then loop to put their
+ * contents into an array of Objects.
+ * @param {String} dataDir - Directory containing the files to load.
+ * @return {Array} - Each element of the array is an Object based on the
+ *                   contents of each file loaded.
+ */
+dbLoader.getDataFromDirectory = function (dataDir) {
+    var contentArray = [];
+    var files = fs.readdirSync(dataDir);
+    files.forEach(function (aFile) {
+        if (aFile.endsWith(".json")) {
+            var fileContent = fs.readFileSync(dataDir + "/" + aFile, "utf-8");
+            contentArray.push(JSON.parse(fileContent));
+        }
+    });
+    return contentArray;
+};
+
 // Get the snapsets Prefs Safes.
 var snapsetsPromise = fluid.promise();
 var getSnapSetsResponse = dbLoader.createResponseHandler(
@@ -234,10 +243,36 @@ snapsetsPromise.then(function () { getGpiiKeysRequest.end(); });
 
 // Batch delete snapset Prefs Safes and their GPII Keys.
 var batchDeletePromise = fluid.promise();
-var batchDeleteResponse = dbLoader.createResponseHandler(dbLoader.procesBatchDelete, batchDeletePromise);
-var execBatchDelete = dbLoader.doBatchDelete(batchDeleteResponse);
+var batchDeleteResponse = dbLoader.createResponseHandler(
+    function () { fluid.log("Snapset Prefs Safes and associated GPII Keys deleted."); },
+    batchDeletePromise
+);
+var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
+var execBatchDelete = dbLoader.createBulkDocsRequest(docsToRemove, batchDeleteResponse);
 gpiiKeysPromise.then(execBatchDelete);
 
-// Done.
-batchDeletePromise.then(dbLoader.bulkDeletionComplete);
+// ==========
+// Load the snapset PrefsSafes, their GPII Keys, and client credentials from disk.
+
+// Load the static data
+var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
+var staticDataPromise = fluid.promise();
+var staticPostResponse = dbLoader.createResponseHandler(
+    function () { fluid.log ("Bulk loading of static data from '" + dbLoader.staticDataDir + "'"); },
+    staticDataPromise
+);
+var execStaticDataRequest = dbLoader.createBulkDocsRequest(staticData, staticPostResponse);
+batchDeletePromise.then(execStaticDataRequest);
+
+// Load the build data
+var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
+var buildDataPromise = fluid.promise();
+var buildPostResponse = dbLoader.createResponseHandler(
+    function () { fluid.log ("Bulk loading of build data from '" + dbLoader.buildDataDir + "'"); },
+    buildDataPromise
+);
+var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildPostResponse);
+staticDataPromise.then(execBuildDataRequest);
+buildDataPromise.then(function () { fluid.log("Done."); });
+
 

From 0e416004227b0a5419bfe5d36327f58d0ce11e18 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 24 Jul 2018 09:36:56 -0400
Subject: [PATCH 12/41] GPII-3138:  Delete and reload snapsets into the
 database

Fixed bug in merging the data from the static and build directories
when forming the array of documents to post to the database.
---
 scripts/deleteAndLoadSnapsets.js | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 7c7f658ff..df5897164 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -31,7 +31,7 @@ var dbLoader = gpii.dataLoader;
 
 // Handle command line
 if (process.argv.length !== 5) {
-    fluid.log("Usage: node deleteSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR");
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR");
     process.exit(1);
 }
 dbLoader.couchDbUrl = process.argv[2];
@@ -132,20 +132,13 @@ dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler) {
         };
         var batchPostData = JSON.stringify({"docs": dataToPost});
         postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
-        var batchDeleteRequest = http.request(postOptions, responseHandler);
-        batchDeleteRequest.write(batchPostData);
-        batchDeleteRequest.end();
-        return batchDeleteRequest;
+        var batchDocsRequest = http.request(postOptions, responseHandler);
+        batchDocsRequest.write(batchPostData);
+        batchDocsRequest.end();
+        return batchDocsRequest;
     };
 };
 
-/**
- * Log that the snapsets (Prefs Safes and GPII Keys) have been deleted.
- */
-dbLoader.bulkDeletionComplete = function () {
-    fluid.log ("\tBulk deletion completed.");
-};
-
 /**
  * Generate a response handler, setting up the given promise to resolve/reject
  * at the correct time.
@@ -207,7 +200,7 @@ dbLoader.getDataFromDirectory = function (dataDir) {
     files.forEach(function (aFile) {
         if (aFile.endsWith(".json")) {
             var fileContent = fs.readFileSync(dataDir + "/" + aFile, "utf-8");
-            contentArray.push(JSON.parse(fileContent));
+            contentArray = contentArray.concat(JSON.parse(fileContent));
         }
     });
     return contentArray;

From 22d6605c3b761292235114ea752896934b011979 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 25 Jul 2018 11:13:45 -0400
Subject: [PATCH 13/41] GPII-3138:  Delete and load snapsets Prefs Safes.

Fixed logic error in when to create the array of snapset Prefs
Safes and their associated GPII Keys prior to making the bulk docs
request.
---
 scripts/deleteAndLoadSnapsets.js | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index df5897164..b296ea53c 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -59,6 +59,7 @@ fluid.log("BUILD_DATA_DIR: '" + dbLoader.buildDataDir + "'");
  * them to an array of records to remove.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving the snapset PrefsSafes records
+ * @return {Array} - The snapset PrefsSafes records marked for deletion.
  */
 dbLoader.processSnapsets = function (responseString) {
     fluid.log("Processing the snapset Prefs Safes records...");
@@ -68,6 +69,7 @@ dbLoader.processSnapsets = function (responseString) {
         dbLoader.snapsetPrefsSafes.push(aSnapset.value);
     });
     fluid.log("\tSnapset Prefs Safes marked for deletion.");
+    return dbLoader.snapsetPrefsSafes;
 };
 
 /**
@@ -75,6 +77,7 @@ dbLoader.processSnapsets = function (responseString) {
  * them for deletion, and add them to array of records to delete.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving all the GPII Keys.
+ * @return {Array} - The GPII Key records marked for deletion.
  */
 dbLoader.processGpiiKeys = function (responseString) {
     fluid.log("Processing the GPII Keys...");
@@ -83,6 +86,7 @@ dbLoader.processGpiiKeys = function (responseString) {
         gpiiKeyRecords, dbLoader.snapsetPrefsSafes
     );
     fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
+    return dbLoader.gpiiKeys;
 };
 
 /**
@@ -110,6 +114,19 @@ dbLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets)
     return gpiiKeysToDelete;
 };
 
+/**
+ * Utility to wrap all the pieces to make a bulk documents deletion request
+ * for the snapset Prefs Safes and their associated GPII keys.  Its intended use
+ * is the parameter of the appropriate promise.then() call.
+ */
+dbLoader.doBatchDelete = function () {
+    var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
+    var execBatchDelete = dbLoader.createBulkDocsRequest(
+        docsToRemove, dbLoader.batchDeleteResponse
+    );
+    execBatchDelete();
+};
+
 /**
  * Create a function that makes a bulk docs POST request using the given data.
  * @param {Object} dataToPost - JSON data to POST and process in bulk.
@@ -232,17 +249,16 @@ var getGpiiKeysRequest = dbLoader.queryDatabase(
     getGpiiKeysResponse,
     "Error requesting GPII Keys: "
 );
+
 snapsetsPromise.then(function () { getGpiiKeysRequest.end(); });
 
 // Batch delete snapset Prefs Safes and their GPII Keys.
 var batchDeletePromise = fluid.promise();
-var batchDeleteResponse = dbLoader.createResponseHandler(
+dbLoader.batchDeleteResponse = dbLoader.createResponseHandler(
     function () { fluid.log("Snapset Prefs Safes and associated GPII Keys deleted."); },
     batchDeletePromise
 );
-var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
-var execBatchDelete = dbLoader.createBulkDocsRequest(docsToRemove, batchDeleteResponse);
-gpiiKeysPromise.then(execBatchDelete);
+gpiiKeysPromise.then(dbLoader.doBatchDelete);
 
 // ==========
 // Load the snapset PrefsSafes, their GPII Keys, and client credentials from disk.
@@ -268,4 +284,3 @@ var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildPostRe
 staticDataPromise.then(execBuildDataRequest);
 buildDataPromise.then(function () { fluid.log("Done."); });
 
-

From 574236fb37f9091b03eff0b2ed389a43c2f72d75 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 25 Jul 2018 11:28:37 -0400
Subject: [PATCH 14/41] GPII-3138:  Delete and load snapsets Prefs Safes into
 the database.

Improved error handling.
---
 scripts/deleteAndLoadSnapsets.js | 42 ++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index b296ea53c..1d2a028db 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -176,8 +176,23 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
             responseString += chunk;
         });
         response.on("end", function () {
-            var value = handleEnd(responseString);
-            promise.resolve(value);
+            if (response.statusCode >= 400) {   // error
+                var fullErrorMsg = errorMsg +
+                                   response.statusCode + " - " +
+                                   response.statusMessage;
+                // Document-not-found or 404 errors include a reason in the
+                // response.
+                // http://docs.couchdb.org/en/stable/api/basics.html#http-status-codes
+                if (response.statusCode === 404) {
+                    fullErrorMsg = fullErrorMsg + ", " +
+                                   JSON.parse(responseString).reason;
+                }
+                promise.reject(fullErrorMsg);
+            }
+            else {
+                var value = handleEnd(responseString);
+                promise.resolve(value);
+            }
         });
         response.on("error", function (e) {
             fluid.log(errorMsg + e.message);
@@ -186,6 +201,18 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
     };
 };
 
+/**
+ * Quit the whole process because a request of the database has failed, and log
+ * the error.  Use this function when the failure has not actually modified the
+ * database; for example, when getting all the current snapset Prefs Safes.  If
+ * that failed, that database is as it was, but there is no point in continuing.
+ * @param {String} errorMsg - The reason why database access failed.
+ */
+dbLoader.bail = function (errorMsg) {
+    fluid.log (errorMsg);
+    process.exit(1);
+};
+
 /**
  * General mechanism to create a database request, set up an error handler and
  * return.  It is up to the caller to trigger the request by calling its end()
@@ -249,8 +276,7 @@ var getGpiiKeysRequest = dbLoader.queryDatabase(
     getGpiiKeysResponse,
     "Error requesting GPII Keys: "
 );
-
-snapsetsPromise.then(function () { getGpiiKeysRequest.end(); });
+snapsetsPromise.then(function () { getGpiiKeysRequest.end(); }, dbLoader.bail);
 
 // Batch delete snapset Prefs Safes and their GPII Keys.
 var batchDeletePromise = fluid.promise();
@@ -258,7 +284,7 @@ dbLoader.batchDeleteResponse = dbLoader.createResponseHandler(
     function () { fluid.log("Snapset Prefs Safes and associated GPII Keys deleted."); },
     batchDeletePromise
 );
-gpiiKeysPromise.then(dbLoader.doBatchDelete);
+gpiiKeysPromise.then(dbLoader.doBatchDelete, dbLoader.bail);
 
 // ==========
 // Load the snapset PrefsSafes, their GPII Keys, and client credentials from disk.
@@ -271,7 +297,7 @@ var staticPostResponse = dbLoader.createResponseHandler(
     staticDataPromise
 );
 var execStaticDataRequest = dbLoader.createBulkDocsRequest(staticData, staticPostResponse);
-batchDeletePromise.then(execStaticDataRequest);
+batchDeletePromise.then(execStaticDataRequest, dbLoader.bail);
 
 // Load the build data
 var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
@@ -281,6 +307,6 @@ var buildPostResponse = dbLoader.createResponseHandler(
     buildDataPromise
 );
 var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildPostResponse);
-staticDataPromise.then(execBuildDataRequest);
-buildDataPromise.then(function () { fluid.log("Done."); });
+staticDataPromise.then(execBuildDataRequest, dbLoader.bail);
+buildDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
 

From d6547d3843c104ef2ac3580703ee79fc7551b830 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 25 Jul 2018 11:52:46 -0400
Subject: [PATCH 15/41] GPII-3138:  Delete and load snapsets Prefs Safes into
 the database.

Added option to just delete the current snapset Prefs Safes and
their associated GPII Keys without loading the static and build data.
Also modified the loading of the static and build data such that it
is done is one go (one build docs request).
---
 scripts/deleteAndLoadSnapsets.js | 54 +++++++++++++++++---------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 1d2a028db..8b8d2202b 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -30,13 +30,18 @@ fluid.setLogging(fluid.logLevel.INFO);
 var dbLoader = gpii.dataLoader;
 
 // Handle command line
-if (process.argv.length !== 5) {
-    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR");
+if (process.argv.length < 5) {
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR [--justDelete]");
     process.exit(1);
 }
 dbLoader.couchDbUrl = process.argv[2];
 dbLoader.staticDataDir = process.argv[3];
 dbLoader.buildDataDir = process.argv[4];
+if (process.argv.length > 5 && process.argv[5] === "--justDelete") { // for debugging.
+    dbLoader.justDelete = true;
+} else {
+    dbLoader.justDelete = false;
+}
 
 dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
 dbLoader.gpiiKeysViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
@@ -286,27 +291,26 @@ dbLoader.batchDeleteResponse = dbLoader.createResponseHandler(
 );
 gpiiKeysPromise.then(dbLoader.doBatchDelete, dbLoader.bail);
 
-// ==========
-// Load the snapset PrefsSafes, their GPII Keys, and client credentials from disk.
-
-// Load the static data
-var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
-var staticDataPromise = fluid.promise();
-var staticPostResponse = dbLoader.createResponseHandler(
-    function () { fluid.log ("Bulk loading of static data from '" + dbLoader.staticDataDir + "'"); },
-    staticDataPromise
-);
-var execStaticDataRequest = dbLoader.createBulkDocsRequest(staticData, staticPostResponse);
-batchDeletePromise.then(execStaticDataRequest, dbLoader.bail);
-
-// Load the build data
-var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
-var buildDataPromise = fluid.promise();
-var buildPostResponse = dbLoader.createResponseHandler(
-    function () { fluid.log ("Bulk loading of build data from '" + dbLoader.buildDataDir + "'"); },
-    buildDataPromise
-);
-var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildPostResponse);
-staticDataPromise.then(execBuildDataRequest, dbLoader.bail);
-buildDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
+if (dbLoader.justDelete) {
+    batchDeletePromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
+} else {
+    // ==========
+    // Load the snapset PrefsSafes, their GPII Keys, and client credentials from
+    // disk, i.e. load the static and build data
+    var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
+    var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
+    var allData = staticData.concat(buildData);
+    var allDataPromise = fluid.promise();
+    var allDataResponse = dbLoader.createResponseHandler(
+        function () {
+            fluid.log ("Bulk loading of static and build data from '" +
+                        dbLoader.staticDataDir + ", and '" +
+                        dbLoader.buildDataDir + "', respectively");
+        },
+        allDataPromise
+    );
+    var execAllDataRequest = dbLoader.createBulkDocsRequest(allData, allDataResponse);
+    batchDeletePromise.then(execAllDataRequest, dbLoader.bail);
+    allDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
+}
 

From 8520379cbb8f281414c583371ad766eb3e570c56 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 25 Jul 2018 15:57:50 -0400
Subject: [PATCH 16/41] GPII-3138:  Delete and load snapsets into the database

Reorganized order to ensure that the static data (the views) are
loaded into a fresh database.
---
 scripts/deleteAndLoadSnapsets.js | 45 +++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 8b8d2202b..1bbc5f949 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -255,7 +255,22 @@ dbLoader.getDataFromDirectory = function (dataDir) {
     return contentArray;
 };
 
-// Get the snapsets Prefs Safes.
+// Load the static data first.  If the database is fresh, there won't be any
+// views to use to find the snapset Prefs Safes.  This ensures they are loaded
+// regardless.  If the data base already has these views, then this is
+// essentially a no-op.
+var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
+var staticDataPromise = fluid.promise();
+var staticDataResponse = dbLoader.createResponseHandler(
+    function () {
+        fluid.log("Loading static data from '" + dbLoader.staticDataDir + "'");
+    },
+    staticDataPromise
+);
+var execStaticDataRequest = dbLoader.createBulkDocsRequest(staticData, staticDataResponse);
+execStaticDataRequest();
+
+// Secondly, get the snapsets Prefs Safes.
 var snapsetsPromise = fluid.promise();
 var getSnapSetsResponse = dbLoader.createResponseHandler(
     dbLoader.processSnapsets,
@@ -267,9 +282,9 @@ var snapSetsRequest = dbLoader.queryDatabase(
     getSnapSetsResponse,
     "Error requesting snapsets Prefs Safes: "
 );
-snapSetsRequest.end();
+staticDataPromise.then(function () { snapSetsRequest.end(); }, dbLoader.bail);
 
-// Get the associated GPII Keys.
+// Thirdly, the associated GPII Keys.
 var gpiiKeysPromise = fluid.promise();
 var getGpiiKeysResponse = dbLoader.createResponseHandler(
     dbLoader.processGpiiKeys,
@@ -283,7 +298,7 @@ var getGpiiKeysRequest = dbLoader.queryDatabase(
 );
 snapsetsPromise.then(function () { getGpiiKeysRequest.end(); }, dbLoader.bail);
 
-// Batch delete snapset Prefs Safes and their GPII Keys.
+// Next, delete the snapset Prefs Safes and their GPII Keys in batch.
 var batchDeletePromise = fluid.promise();
 dbLoader.batchDeleteResponse = dbLoader.createResponseHandler(
     function () { fluid.log("Snapset Prefs Safes and associated GPII Keys deleted."); },
@@ -294,23 +309,17 @@ gpiiKeysPromise.then(dbLoader.doBatchDelete, dbLoader.bail);
 if (dbLoader.justDelete) {
     batchDeletePromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
 } else {
-    // ==========
-    // Load the snapset PrefsSafes, their GPII Keys, and client credentials from
-    // disk, i.e. load the static and build data
-    var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
+    // Finally, load the latest snapsets data from the build data.
     var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
-    var allData = staticData.concat(buildData);
-    var allDataPromise = fluid.promise();
-    var allDataResponse = dbLoader.createResponseHandler(
+    var buildDataPromise = fluid.promise();
+    var buildDataResponse = dbLoader.createResponseHandler(
         function () {
-            fluid.log ("Bulk loading of static and build data from '" +
-                        dbLoader.staticDataDir + ", and '" +
-                        dbLoader.buildDataDir + "', respectively");
+            fluid.log ("Bulk loading of build data from '" + dbLoader.buildDataDir + "'");
         },
-        allDataPromise
+        buildDataPromise
     );
-    var execAllDataRequest = dbLoader.createBulkDocsRequest(allData, allDataResponse);
-    batchDeletePromise.then(execAllDataRequest, dbLoader.bail);
-    allDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
+    var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildDataResponse);
+    batchDeletePromise.then(execBuildDataRequest, dbLoader.bail);
+    buildDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
 }
 

From 0181c420b79524c0d5cef947cb0ecb1709baad55 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Fri, 27 Jul 2018 16:14:08 -0400
Subject: [PATCH 17/41] GPII-3138:  Delete and load snapsets into the database

- modified convertPrefs.js to create user vs. snapset Prefs Safes as
needed,
- modified package.json postinstall script to create the user and
snapset Prefs Safes and place them in "snapset" vs. "user" subfolders,
- modified testing code and configuration files to pick up the relevant
type of Prefs Safes from their new locations.
---
 .../configs/gpii.pouchManager.config.base.json5             | 4 ++--
 gpii/node_modules/testing/src/PouchTestCaseHolder.js        | 4 ++--
 package.json                                                | 2 +-
 scripts/convertPrefs.js                                     | 6 +++---
 scripts/deleteAndLoadSnapsets.js                            | 1 -
 testData/dbData/README.txt                                  | 6 ++++--
 6 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5 b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
index af0dfcfe0..e62835e67 100644
--- a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
+++ b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
@@ -18,8 +18,8 @@
                     "data": [
                         "%gpii-universal/testData/dbData/clientCredentials.json",
                         "%gpii-universal/testData/dbData/gpiiAppInstallationClients.json",
-                        "%gpii-universal/build/dbData/gpiiKeys.json",
-                        "%gpii-universal/build/dbData/prefsSafes.json",
+                        "%gpii-universal/build/dbData/user/gpiiKeys.json",
+                        "%gpii-universal/build/dbData/user/prefsSafes.json",
                         "%gpii-universal/testData/dbData/views.json"
                     ]
                 }
diff --git a/gpii/node_modules/testing/src/PouchTestCaseHolder.js b/gpii/node_modules/testing/src/PouchTestCaseHolder.js
index 1ed4d6095..55d5a91cf 100644
--- a/gpii/node_modules/testing/src/PouchTestCaseHolder.js
+++ b/gpii/node_modules/testing/src/PouchTestCaseHolder.js
@@ -47,8 +47,8 @@ fluid.defaults("gpii.test.pouch.pouchTestCaseHolder", {
                     "%gpii-universal/tests/data/dbData/gpiiKeys.json",
                     "%gpii-universal/build/tests/dbData/gpiiKeys.json",
                     "%gpii-universal/build/tests/dbData/prefsSafes.json",
-                    "%gpii-universal/build/dbData/gpiiKeys.json",
-                    "%gpii-universal/build/dbData/prefsSafes.json",
+                    "%gpii-universal/build/dbData/user/gpiiKeys.json",
+                    "%gpii-universal/build/dbData/user/prefsSafes.json",
                     "%gpii-universal/testData/dbData/views.json"
                 ]
             }
diff --git a/package.json b/package.json
index c3d0b82eb..4bc2a52b2 100644
--- a/package.json
+++ b/package.json
@@ -77,6 +77,6 @@
         "test:vagrantProduction": "vagrant ssh -c 'cd /home/vagrant/sync/universal; DISPLAY=:0; ./scripts/vagrantCloudBasedContainers.sh'",
         "posttest": "node node_modules/nyc/bin/nyc.js report -r text-summary -r html --report-dir reports --temp-directory coverage",
         "start": "node gpii.js",
-        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/ && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/"
+        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset && node scripts/convertPrefs.js testData/preferences/ build/dbData/user/ user && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/ user"
     }
 }
diff --git a/scripts/convertPrefs.js b/scripts/convertPrefs.js
index 3aa60a2e9..8fec59016 100644
--- a/scripts/convertPrefs.js
+++ b/scripts/convertPrefs.js
@@ -25,14 +25,14 @@ var fs = require("fs"),
 
 var inputDir = process.argv[2];
 var targetDir = process.argv[3];
+var prefsSafeType = process.argv[4];
 
 var prefsSafes = [];
 var gpiiKeys = [];
-var count = 0;
 
 var filenames = fs.readdirSync(inputDir);
 
-console.log("Converting preferences data in the source directory " + inputDir + " to the target directory " + targetDir + " ...");
+console.log("Converting preferences data in the source directory " + inputDir + " to the target directory " + targetDir + " as " + prefsSafeType + " Prefs Safes ...");
 
 // Read and loop thru json5 files in the input directory
 rimraf(targetDir, function () {
@@ -61,7 +61,7 @@ rimraf(targetDir, function () {
                     "_id": prefsSafeId,
                     "type": "prefsSafe",
                     "schemaVersion": "0.1",
-                    "prefsSafeType": "snapset",
+                    "prefsSafeType": prefsSafeType,
                     "name": gpiiKey,
                     "password": null,
                     "email": null,
diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 1bbc5f949..e4c36fd16 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -322,4 +322,3 @@ if (dbLoader.justDelete) {
     batchDeletePromise.then(execBuildDataRequest, dbLoader.bail);
     buildDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
 }
-
diff --git a/testData/dbData/README.txt b/testData/dbData/README.txt
index 70f4fbcb7..2c4c47200 100644
--- a/testData/dbData/README.txt
+++ b/testData/dbData/README.txt
@@ -4,8 +4,10 @@ This folder contains the DB data that is used for running GPII in a development
 
 Each data file is corresponding to a document structure specified in the documentation of [GPII Data Model](https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences):
 
-* %universal/build/dbData/gpiiKeys.json: Contains all GPII keys.
-* %universal/build/dbData/prefsSafes.json: Contains all preferences safes corresponding to GPII keys.
+* %universal/build/dbData/snapset/gpiiKeys.json: Contains all GPII keys for snapset preferences safes.
+* %universal/build/dbData/snapset/prefsSafes.json: Contains all snapset preferences safes corresponding to snapset GPII keys.
+* %universal/build/dbData/user/gpiiKeys.json: Contains all GPII keys for user preferences safes.
+* %universal/build/dbData/user/prefsSafes.json: Contains all user preferences safes corresponding to user GPII keys.
 * clientCredentials.json: Contains OAuth2 client credentials.
 * gpiiAppInstallationClients.json: Contains the information of all registered GPII app installations.
 * views.json: The PouchDB/CouchDB view functions.

From 9bb64afb883a913303a65dc759a040129e5e14b2 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 30 Jul 2018 10:56:46 -0400
Subject: [PATCH 18/41] GPII-3138:  Update snapsets into the database

- updated vagrantCloudBasedContainers.sh script to properly reference
snapset Prefs Safes,
- fixed bug so that pouchdb tests use snapset instead of user Prefs Safes,
- removed unused database view.
---
 .../pouchManager/configs/gpii.pouchManager.config.base.json5  | 4 ++--
 scripts/vagrantCloudBasedContainers.sh                        | 2 +-
 testData/dbData/views.json                                    | 3 ---
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5 b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
index e62835e67..19484578c 100644
--- a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
+++ b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json5
@@ -18,8 +18,8 @@
                     "data": [
                         "%gpii-universal/testData/dbData/clientCredentials.json",
                         "%gpii-universal/testData/dbData/gpiiAppInstallationClients.json",
-                        "%gpii-universal/build/dbData/user/gpiiKeys.json",
-                        "%gpii-universal/build/dbData/user/prefsSafes.json",
+                        "%gpii-universal/build/dbData/snapset/gpiiKeys.json",
+                        "%gpii-universal/build/dbData/snapset/prefsSafes.json",
                         "%gpii-universal/testData/dbData/views.json"
                     ]
                 }
diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index c7f23a860..f52511b4f 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -40,7 +40,7 @@ fi
 
 UNIVERSAL_DIR="/home/vagrant/sync/universal"
 STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
-BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData"
+BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData/snapset"
 
 DATALOADER_IMAGE="gpii/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
diff --git a/testData/dbData/views.json b/testData/dbData/views.json
index b6ab3dc0c..f2fecc164 100644
--- a/testData/dbData/views.json
+++ b/testData/dbData/views.json
@@ -14,9 +14,6 @@
             "findSnapsetPrefsSafes": {
                 "map": "function(doc) {if (doc.type === 'prefsSafe' && doc.prefsSafeType === 'snapset') {emit(doc._id, doc)}}"
             },
-            "findGpiiKeysByPrefsSafeId": {
-                "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc.prefsSafeId, doc)}}"
-            },
             "findAllGpiiKeys": {
                 "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc._id, doc)}}"
             }

From a4541ba8a3ccb9aec375c1180ca5ab6a65422980 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 30 Jul 2018 15:14:18 -0400
Subject: [PATCH 19/41] GPII-3138:  Update snapsets into the database

Special version of vagrantCloudBasedContainers.sh that loads the
data docker image from Joseph's docker hub.  That docker image
contains the logic that his branch of gpii-dataloader uses to create
the docker image.  Use this until such time as his GPII-3138 branch
of gpii-dataloader is merged into master.
---
 scripts/vagrantCloudBasedContainers.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index f52511b4f..49bcc8d4f 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -42,7 +42,7 @@ UNIVERSAL_DIR="/home/vagrant/sync/universal"
 STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
 BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData/snapset"
 
-DATALOADER_IMAGE="gpii/gpii-dataloader"
+DATALOADER_IMAGE="herrclown/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
 DATASOURCE_HOSTNAME="http://couchdb"
 

From f4c41a6e3a1b1ceae55a0bec568c05dceda2a0d3 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 31 Jul 2018 16:09:35 -0400
Subject: [PATCH 20/41] GPII-3138:  Update snapsets into the database

New demo user preferences added to the database as 'user' PrefsSafes.
---
 package.json                                  |  2 +-
 scripts/deleteAndLoadSnapsets.js              |  7 ++-
 scripts/vagrantCloudBasedContainers.sh        |  3 +-
 .../preferences/demoUserPrefs/quint.json5     | 52 +++++++++++++++++++
 testData/preferences/demoUserPrefs/roy.json5  | 51 ++++++++++++++++++
 5 files changed, 111 insertions(+), 4 deletions(-)
 create mode 100644 testData/preferences/demoUserPrefs/quint.json5
 create mode 100644 testData/preferences/demoUserPrefs/roy.json5

diff --git a/package.json b/package.json
index 4bc2a52b2..dc3a08f01 100644
--- a/package.json
+++ b/package.json
@@ -77,6 +77,6 @@
         "test:vagrantProduction": "vagrant ssh -c 'cd /home/vagrant/sync/universal; DISPLAY=:0; ./scripts/vagrantCloudBasedContainers.sh'",
         "posttest": "node node_modules/nyc/bin/nyc.js report -r text-summary -r html --report-dir reports --temp-directory coverage",
         "start": "node gpii.js",
-        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset && node scripts/convertPrefs.js testData/preferences/ build/dbData/user/ user && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/ user"
+        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset && node scripts/convertPrefs.js testData/preferences/ build/dbData/user/ user && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/ user && node scripts/convertPrefs.js testData/preferences/demoUserPrefs/ build/dbData/demouser/ user"
     }
 }
diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index e4c36fd16..328330669 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -31,13 +31,14 @@ var dbLoader = gpii.dataLoader;
 
 // Handle command line
 if (process.argv.length < 5) {
-    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR [--justDelete]");
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
     process.exit(1);
 }
 dbLoader.couchDbUrl = process.argv[2];
 dbLoader.staticDataDir = process.argv[3];
 dbLoader.buildDataDir = process.argv[4];
-if (process.argv.length > 5 && process.argv[5] === "--justDelete") { // for debugging.
+dbLoader.demoUserDir = process.argv[5];
+if (process.argv.length > 6 && process.argv[6] === "--justDelete") { // for debugging.
     dbLoader.justDelete = true;
 } else {
     dbLoader.justDelete = false;
@@ -311,6 +312,8 @@ if (dbLoader.justDelete) {
 } else {
     // Finally, load the latest snapsets data from the build data.
     var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
+    var demoUserData = dbLoader.getDataFromDirectory(dbLoader.demoUserDir);
+    buildData = buildData.concat(demoUserData);
     var buildDataPromise = fluid.promise();
     var buildDataResponse = dbLoader.createResponseHandler(
         function () {
diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index 49bcc8d4f..c7361a78e 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -41,6 +41,7 @@ fi
 UNIVERSAL_DIR="/home/vagrant/sync/universal"
 STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
 BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData/snapset"
+BUILD_DEMOUSER_DIR="$UNIVERSAL_DIR/build/dbData/demouser"
 
 DATALOADER_IMAGE="herrclown/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
@@ -82,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -v $BUILD_DEMOUSER_DIR:/build_data/demouser -e $BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.
diff --git a/testData/preferences/demoUserPrefs/quint.json5 b/testData/preferences/demoUserPrefs/quint.json5
new file mode 100644
index 000000000..484a0fbe8
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/quint.json5
@@ -0,0 +1,52 @@
+{
+    "flat": {
+        "name": "Quint",
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {
+                    "http://registry.gpii.net/common/language": "es-ES",
+                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
+                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
+                        "Options.PrimarySynthesizer" : "eloq",
+                        "options.SayAllIndicateCaps" : false,
+                        "ENU-Global.Rate" : 50,
+                        "ENU-Global.Pitch" : 75,
+                        "ENU-Global.Volume" : 100,
+                        "ENU-Global.Punctuation" : 2,
+                        "ENU-JAWSCursor.Rate": 50,
+                        "ENU-Keyboard.Rate": 50,
+                        "ENU-MenuAndDialog.Rate": 50,
+                        "ENU-Message.Rate": 50,
+                        "ENU-PCCursor.Rate": 50,
+                        "ENU-JAWSCursor.Pitch": 75,
+                        "ENU-Keyboard.Pitch": 75,
+                        "ENU-MenuAndDialog.Pitch": 75,
+                        "ENU-Message.Pitch": 75,
+                        "ENU-PCCursor.Pitch": 75,
+                        "ENU-JAWSCursor.Volume": 100,
+                        "ENU-Keyboard.Volume": 100,
+                        "ENU-MenuAndDialog.Volume": 100,
+                        "ENU-Message.Volume": 100,
+                        "ENU-PCCursor.Volume": 100,
+                        "ENU-JAWSCursor.Punctuation": 2,
+                        "ENU-Keyboard.Punctuation": 2,
+                        "ENU-MenuAndDialog.Punctuation": 2,
+                        "ENU-Message.Punctuation": 2,
+                        "ENU-PCCursor.Punctuation": 2,
+                        "ENU-Global.SynthLangString": "Castilian Spanish",
+                        "ENU-JAWSCursor.SynthLangString": "Castilian Spanish",
+                        "ENU-Keyboard.SynthLangString": "Castilian Spanish",
+                        "ENU-MenuAndDialog.SynthLangString": "Castilian Spanish",
+                        "ENU-Message.SynthLangString": "Castilian Spanish",
+                        "ENU-PCCursor.SynthLangString": "Castilian Spanish",
+                        "options.TypingEcho": 3,
+                        "options.SayAllMode": 0,
+                        "Braille.BrailleMode": 0,
+                        "options.SayAllIgnoreShiftKeys": true
+                    }
+                }
+            }
+        }
+    }
+}   
\ No newline at end of file
diff --git a/testData/preferences/demoUserPrefs/roy.json5 b/testData/preferences/demoUserPrefs/roy.json5
new file mode 100644
index 000000000..b2d2bf743
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/roy.json5
@@ -0,0 +1,51 @@
+{
+    "flat": {
+        "name": "Roy",
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {
+                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
+                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
+                        "Options.PrimarySynthesizer" : "eloq",
+                        "options.SayAllIndicateCaps" : false,
+                        "ENU-Global.Rate" : 100,
+                        "ENU-Global.Pitch" : 65,
+                        "ENU-Global.Volume" : 100,
+                        "ENU-Global.Punctuation" : 2,
+                        "ENU-JAWSCursor.Rate": 100,
+                        "ENU-Keyboard.Rate": 100,
+                        "ENU-MenuAndDialog.Rate": 100,
+                        "ENU-Message.Rate": 100,
+                        "ENU-PCCursor.Rate": 100,
+                        "ENU-JAWSCursor.Pitch": 65,
+                        "ENU-Keyboard.Pitch": 65,
+                        "ENU-MenuAndDialog.Pitch": 65,
+                        "ENU-Message.Pitch": 65,
+                        "ENU-PCCursor.Pitch": 65,
+                        "ENU-JAWSCursor.Volume": 100,
+                        "ENU-Keyboard.Volume": 100,
+                        "ENU-MenuAndDialog.Volume": 100,
+                        "ENU-Message.Volume": 100,
+                        "ENU-PCCursor.Volume": 100,
+                        "ENU-JAWSCursor.Punctuation": 2,
+                        "ENU-Keyboard.Punctuation": 2,
+                        "ENU-MenuAndDialog.Punctuation": 2,
+                        "ENU-Message.Punctuation": 2,
+                        "ENU-PCCursor.Punctuation": 2,
+                        "ENU-Global.SynthLangString": "American English",
+                        "ENU-JAWSCursor.SynthLangString": "American English",
+                        "ENU-Keyboard.SynthLangString": "American English",
+                        "ENU-MenuAndDialog.SynthLangString": "American English",
+                        "ENU-Message.SynthLangString": "American English",
+                        "ENU-PCCursor.SynthLangString": "American English",
+                        "options.TypingEcho": 3,
+                        "options.SayAllMode": 0,
+                        "Braille.BrailleMode": 0,
+                        "options.SayAllIgnoreShiftKeys": true
+                    }
+                }
+            }
+        }
+    }
+}   
\ No newline at end of file

From 6723b6e0f3c17261e621c3607d46a82760d97efe Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 31 Jul 2018 16:21:58 -0400
Subject: [PATCH 21/41] GPII-3138:  Update snapsets into the database

Fixed global variable name when running docker image.
---
 scripts/vagrantCloudBasedContainers.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index c7361a78e..340ca8981 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -83,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -v $BUILD_DEMOUSER_DIR:/build_data/demouser -e $BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -v $BUILD_DEMOUSER_DIR:/build_data/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From 2e0bb55ff714164c2580c3fe598ec7d149674579 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 31 Jul 2018 16:49:52 -0400
Subject: [PATCH 22/41] GPII-3138:  Update snapsets into the database

Fixed lint errors in the two demo user preferences files.
---
 testData/preferences/demoUserPrefs/quint.json5 | 2 +-
 testData/preferences/demoUserPrefs/roy.json5   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/testData/preferences/demoUserPrefs/quint.json5 b/testData/preferences/demoUserPrefs/quint.json5
index 484a0fbe8..5a9fd7844 100644
--- a/testData/preferences/demoUserPrefs/quint.json5
+++ b/testData/preferences/demoUserPrefs/quint.json5
@@ -49,4 +49,4 @@
             }
         }
     }
-}   
\ No newline at end of file
+}
diff --git a/testData/preferences/demoUserPrefs/roy.json5 b/testData/preferences/demoUserPrefs/roy.json5
index b2d2bf743..b4e45e4b9 100644
--- a/testData/preferences/demoUserPrefs/roy.json5
+++ b/testData/preferences/demoUserPrefs/roy.json5
@@ -48,4 +48,4 @@
             }
         }
     }
-}   
\ No newline at end of file
+}

From fae02dabd0fd7523dbfae96ea12ab01ec16bcfc1 Mon Sep 17 00:00:00 2001
From: Cindy Qi Li <cli@ocadu.ca>
Date: Wed, 1 Aug 2018 11:57:36 -0400
Subject: [PATCH 23/41] GPII-3138: Use UUIDs for demo user keys as well as
 creating 20 empty user keys.

---
 ...29bc341-0025-417e-8107-cb567ab419d0.json5} | 105 +++++++++---------
 ...32922e6-569b-404c-9db2-c207544cc959.json5} | 103 ++++++++---------
 ...20558561-dc5b-4973-85f3-9bca975090cb.json5 |   2 +
 ...226722c6-8895-40b4-9297-424b9da9c340.json5 |   2 +
 ...2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5 |   2 +
 ...2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5 |   2 +
 ...2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5 |   2 +
 ...339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5 |   2 +
 ...587ad68c-6470-486b-9828-6ee224ae6902.json5 |   2 +
 ...7009c709-9ce0-441f-9b56-aec7827a1d5c.json5 |   2 +
 ...80489c34-50cd-43db-a1bc-9a50960d5b3a.json5 |   2 +
 ...8060c500-6be9-462a-8ae1-8705be497916.json5 |   2 +
 ...8070295c-d4dc-4b02-9113-ac08f5b0484a.json5 |   2 +
 ...881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5 |   2 +
 ...a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5 |   2 +
 ...b7a438ab-cf2a-472d-9a96-5d87ee187770.json5 |   2 +
 ...c3657704-04f6-4f02-a777-c361f0110876.json5 |   2 +
 ...cd5ca821-6a6e-4734-b956-83546bae1a5f.json5 |   2 +
 ...cead392d-05da-4ecc-9bd3-88c789f94d8c.json5 |   2 +
 ...dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5 |   2 +
 ...e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5 |   2 +
 ...e8c62569-ac93-4c45-8085-f172ca570311.json5 |   2 +
 22 files changed, 145 insertions(+), 103 deletions(-)
 rename testData/preferences/demoUserPrefs/{quint.json5 => 129bc341-0025-417e-8107-cb567ab419d0.json5} (97%)
 rename testData/preferences/demoUserPrefs/{roy.json5 => 132922e6-569b-404c-9db2-c207544cc959.json5} (97%)
 create mode 100644 testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
 create mode 100644 testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
 create mode 100644 testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
 create mode 100644 testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
 create mode 100644 testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
 create mode 100644 testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
 create mode 100644 testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
 create mode 100644 testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
 create mode 100644 testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
 create mode 100644 testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
 create mode 100644 testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
 create mode 100644 testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
 create mode 100644 testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
 create mode 100644 testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
 create mode 100644 testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
 create mode 100644 testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
 create mode 100644 testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
 create mode 100644 testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
 create mode 100644 testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
 create mode 100644 testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5

diff --git a/testData/preferences/demoUserPrefs/quint.json5 b/testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
similarity index 97%
rename from testData/preferences/demoUserPrefs/quint.json5
rename to testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
index 5a9fd7844..539d0cbff 100644
--- a/testData/preferences/demoUserPrefs/quint.json5
+++ b/testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
@@ -1,52 +1,53 @@
-{
-    "flat": {
-        "name": "Quint",
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {
-                    "http://registry.gpii.net/common/language": "es-ES",
-                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
-                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
-                        "Options.PrimarySynthesizer" : "eloq",
-                        "options.SayAllIndicateCaps" : false,
-                        "ENU-Global.Rate" : 50,
-                        "ENU-Global.Pitch" : 75,
-                        "ENU-Global.Volume" : 100,
-                        "ENU-Global.Punctuation" : 2,
-                        "ENU-JAWSCursor.Rate": 50,
-                        "ENU-Keyboard.Rate": 50,
-                        "ENU-MenuAndDialog.Rate": 50,
-                        "ENU-Message.Rate": 50,
-                        "ENU-PCCursor.Rate": 50,
-                        "ENU-JAWSCursor.Pitch": 75,
-                        "ENU-Keyboard.Pitch": 75,
-                        "ENU-MenuAndDialog.Pitch": 75,
-                        "ENU-Message.Pitch": 75,
-                        "ENU-PCCursor.Pitch": 75,
-                        "ENU-JAWSCursor.Volume": 100,
-                        "ENU-Keyboard.Volume": 100,
-                        "ENU-MenuAndDialog.Volume": 100,
-                        "ENU-Message.Volume": 100,
-                        "ENU-PCCursor.Volume": 100,
-                        "ENU-JAWSCursor.Punctuation": 2,
-                        "ENU-Keyboard.Punctuation": 2,
-                        "ENU-MenuAndDialog.Punctuation": 2,
-                        "ENU-Message.Punctuation": 2,
-                        "ENU-PCCursor.Punctuation": 2,
-                        "ENU-Global.SynthLangString": "Castilian Spanish",
-                        "ENU-JAWSCursor.SynthLangString": "Castilian Spanish",
-                        "ENU-Keyboard.SynthLangString": "Castilian Spanish",
-                        "ENU-MenuAndDialog.SynthLangString": "Castilian Spanish",
-                        "ENU-Message.SynthLangString": "Castilian Spanish",
-                        "ENU-PCCursor.SynthLangString": "Castilian Spanish",
-                        "options.TypingEcho": 3,
-                        "options.SayAllMode": 0,
-                        "Braille.BrailleMode": 0,
-                        "options.SayAllIgnoreShiftKeys": true
-                    }
-                }
-            }
-        }
-    }
-}
+// Quint: switches the OS to Spanish, turns on JAWS with a Spanish voice.
+{
+    "flat": {
+        "name": "Quint",
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {
+                    "http://registry.gpii.net/common/language": "es-ES",
+                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
+                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
+                        "Options.PrimarySynthesizer" : "eloq",
+                        "options.SayAllIndicateCaps" : false,
+                        "ENU-Global.Rate" : 50,
+                        "ENU-Global.Pitch" : 75,
+                        "ENU-Global.Volume" : 100,
+                        "ENU-Global.Punctuation" : 2,
+                        "ENU-JAWSCursor.Rate": 50,
+                        "ENU-Keyboard.Rate": 50,
+                        "ENU-MenuAndDialog.Rate": 50,
+                        "ENU-Message.Rate": 50,
+                        "ENU-PCCursor.Rate": 50,
+                        "ENU-JAWSCursor.Pitch": 75,
+                        "ENU-Keyboard.Pitch": 75,
+                        "ENU-MenuAndDialog.Pitch": 75,
+                        "ENU-Message.Pitch": 75,
+                        "ENU-PCCursor.Pitch": 75,
+                        "ENU-JAWSCursor.Volume": 100,
+                        "ENU-Keyboard.Volume": 100,
+                        "ENU-MenuAndDialog.Volume": 100,
+                        "ENU-Message.Volume": 100,
+                        "ENU-PCCursor.Volume": 100,
+                        "ENU-JAWSCursor.Punctuation": 2,
+                        "ENU-Keyboard.Punctuation": 2,
+                        "ENU-MenuAndDialog.Punctuation": 2,
+                        "ENU-Message.Punctuation": 2,
+                        "ENU-PCCursor.Punctuation": 2,
+                        "ENU-Global.SynthLangString": "Castilian Spanish",
+                        "ENU-JAWSCursor.SynthLangString": "Castilian Spanish",
+                        "ENU-Keyboard.SynthLangString": "Castilian Spanish",
+                        "ENU-MenuAndDialog.SynthLangString": "Castilian Spanish",
+                        "ENU-Message.SynthLangString": "Castilian Spanish",
+                        "ENU-PCCursor.SynthLangString": "Castilian Spanish",
+                        "options.TypingEcho": 3,
+                        "options.SayAllMode": 0,
+                        "Braille.BrailleMode": 0,
+                        "options.SayAllIgnoreShiftKeys": true
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/roy.json5 b/testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
similarity index 97%
rename from testData/preferences/demoUserPrefs/roy.json5
rename to testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
index b4e45e4b9..b4642bb85 100644
--- a/testData/preferences/demoUserPrefs/roy.json5
+++ b/testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
@@ -1,51 +1,52 @@
-{
-    "flat": {
-        "name": "Roy",
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {
-                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
-                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
-                        "Options.PrimarySynthesizer" : "eloq",
-                        "options.SayAllIndicateCaps" : false,
-                        "ENU-Global.Rate" : 100,
-                        "ENU-Global.Pitch" : 65,
-                        "ENU-Global.Volume" : 100,
-                        "ENU-Global.Punctuation" : 2,
-                        "ENU-JAWSCursor.Rate": 100,
-                        "ENU-Keyboard.Rate": 100,
-                        "ENU-MenuAndDialog.Rate": 100,
-                        "ENU-Message.Rate": 100,
-                        "ENU-PCCursor.Rate": 100,
-                        "ENU-JAWSCursor.Pitch": 65,
-                        "ENU-Keyboard.Pitch": 65,
-                        "ENU-MenuAndDialog.Pitch": 65,
-                        "ENU-Message.Pitch": 65,
-                        "ENU-PCCursor.Pitch": 65,
-                        "ENU-JAWSCursor.Volume": 100,
-                        "ENU-Keyboard.Volume": 100,
-                        "ENU-MenuAndDialog.Volume": 100,
-                        "ENU-Message.Volume": 100,
-                        "ENU-PCCursor.Volume": 100,
-                        "ENU-JAWSCursor.Punctuation": 2,
-                        "ENU-Keyboard.Punctuation": 2,
-                        "ENU-MenuAndDialog.Punctuation": 2,
-                        "ENU-Message.Punctuation": 2,
-                        "ENU-PCCursor.Punctuation": 2,
-                        "ENU-Global.SynthLangString": "American English",
-                        "ENU-JAWSCursor.SynthLangString": "American English",
-                        "ENU-Keyboard.SynthLangString": "American English",
-                        "ENU-MenuAndDialog.SynthLangString": "American English",
-                        "ENU-Message.SynthLangString": "American English",
-                        "ENU-PCCursor.SynthLangString": "American English",
-                        "options.TypingEcho": 3,
-                        "options.SayAllMode": 0,
-                        "Braille.BrailleMode": 0,
-                        "options.SayAllIgnoreShiftKeys": true
-                    }
-                }
-            }
-        }
-    }
-}
+// Roy: turns on JAWS and speaks relatively fast.
+{
+    "flat": {
+        "name": "Roy",
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {
+                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
+                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
+                        "Options.PrimarySynthesizer" : "eloq",
+                        "options.SayAllIndicateCaps" : false,
+                        "ENU-Global.Rate" : 100,
+                        "ENU-Global.Pitch" : 65,
+                        "ENU-Global.Volume" : 100,
+                        "ENU-Global.Punctuation" : 2,
+                        "ENU-JAWSCursor.Rate": 100,
+                        "ENU-Keyboard.Rate": 100,
+                        "ENU-MenuAndDialog.Rate": 100,
+                        "ENU-Message.Rate": 100,
+                        "ENU-PCCursor.Rate": 100,
+                        "ENU-JAWSCursor.Pitch": 65,
+                        "ENU-Keyboard.Pitch": 65,
+                        "ENU-MenuAndDialog.Pitch": 65,
+                        "ENU-Message.Pitch": 65,
+                        "ENU-PCCursor.Pitch": 65,
+                        "ENU-JAWSCursor.Volume": 100,
+                        "ENU-Keyboard.Volume": 100,
+                        "ENU-MenuAndDialog.Volume": 100,
+                        "ENU-Message.Volume": 100,
+                        "ENU-PCCursor.Volume": 100,
+                        "ENU-JAWSCursor.Punctuation": 2,
+                        "ENU-Keyboard.Punctuation": 2,
+                        "ENU-MenuAndDialog.Punctuation": 2,
+                        "ENU-Message.Punctuation": 2,
+                        "ENU-PCCursor.Punctuation": 2,
+                        "ENU-Global.SynthLangString": "American English",
+                        "ENU-JAWSCursor.SynthLangString": "American English",
+                        "ENU-Keyboard.SynthLangString": "American English",
+                        "ENU-MenuAndDialog.SynthLangString": "American English",
+                        "ENU-Message.SynthLangString": "American English",
+                        "ENU-PCCursor.SynthLangString": "American English",
+                        "options.TypingEcho": 3,
+                        "options.SayAllMode": 0,
+                        "Braille.BrailleMode": 0,
+                        "options.SayAllIgnoreShiftKeys": true
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5 b/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5 b/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5 b/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5 b/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5 b/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5 b/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5 b/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5 b/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5 b/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5 b/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5 b/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5 b/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5 b/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5 b/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5 b/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5 b/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5 b/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5 b/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5 b/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}
diff --git a/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5 b/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
new file mode 100644
index 000000000..9f7a319dd
--- /dev/null
+++ b/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
@@ -0,0 +1,2 @@
+// Empty user key
+{}

From 582b3d72fb2af9ff5daa4d7cb6d1cb2889638b98 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Sun, 5 Aug 2018 15:31:39 -0400
Subject: [PATCH 24/41] GPII-3138:  Update snapsets in the database

Addressed Antranig's review comments:
- Command line arguments and database constants stored in a local
"options" variable with its own accessor,
- Each step of the overall task created/configured by a function,
and then all are executed by an "orchestrate" function.
---
 scripts/deleteAndLoadSnapsets.js | 334 ++++++++++++++++++++-----------
 1 file changed, 217 insertions(+), 117 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 328330669..4c1100c3e 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -8,13 +8,15 @@ You may obtain a copy of the License at
 https://github.com/GPII/universal/blob/master/LICENSE.txt
 */
 
-// This script modifies the preferences data base:
-// 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"),
-// 2. Finds all the GPII Keys associated with each snapset Prefs Safe
-// 3. Deletes the found Prefs Safes and associated GPII Keys
-//
+// This script modifies the preferences database:
+// 1. Loads static data into the database,
+// 2. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
+// 3. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
+// 4. Deletes these Prefs Safes and their associated GPII Keys from the database,
+// 5. Uploads the new Prefs Safes and their GPII Keys to the database,
+// 6. Uploads demo user Prefs Safes and their Keys in to the database, if they are not already present.
 // A sample command that runs this script:
-// node deleteSnapsets.js $COUCHDBURL
+// node deleteAndLoadSnapsets.js $COUCHDBURL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR
 
 "use strict";
 
@@ -27,38 +29,64 @@ var gpii = fluid.registerNamespace("gpii");
 fluid.registerNamespace("gpii.dataLoader");
 fluid.setLogging(fluid.logLevel.INFO);
 
-var dbLoader = gpii.dataLoader;
-
 // Handle command line
 if (process.argv.length < 5) {
     fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
     process.exit(1);
 }
-dbLoader.couchDbUrl = process.argv[2];
-dbLoader.staticDataDir = process.argv[3];
-dbLoader.buildDataDir = process.argv[4];
-dbLoader.demoUserDir = process.argv[5];
-if (process.argv.length > 6 && process.argv[6] === "--justDelete") { // for debugging.
-    dbLoader.justDelete = true;
-} else {
-    dbLoader.justDelete = false;
-}
 
-dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
-dbLoader.gpiiKeysViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
-dbLoader.parsedCouchDbUrl = url.parse(dbLoader.couchDbUrl);
-dbLoader.snapsetPrefsSafes = [];
-dbLoader.gpiiKeys = [];
+var dbLoader = gpii.dataLoader;
 
-fluid.log("COUCHDB_URL: '" +
-    dbLoader.parsedCouchDbUrl.protocol + "//" +
-    dbLoader.parsedCouchDbUrl.hostname + ":" +
-    dbLoader.parsedCouchDbUrl.port +
-    dbLoader.parsedCouchDbUrl.pathname +
-"'");
+/**
+ * Create a set of options for data loader and a function to retreive them.
+ * The options are based on the command line parameters and a set of database
+ * constants.
+ * @param {Array} processArgv - The command line arguments.
+ */
+dbLoader.initOptions = function (processArgv) {
+    var dbOptions = {};
+    dbOptions.couchDbUrl = processArgv[2];
+    dbOptions.staticDataDir = processArgv[3];
+    dbOptions.buildDataDir = processArgv[4];
+    dbOptions.demoUserDir = processArgv[5];
+    if (processArgv.length > 6 && processArgv[6] === "--justDelete") { // for debugging.
+        dbOptions.justDelete = true;
+    } else {
+        dbOptions.justDelete = false;
+    }
 
-fluid.log("STATIC_DATA_DIR: '" + dbLoader.staticDataDir + "'");
-fluid.log("BUILD_DATA_DIR: '" + dbLoader.buildDataDir + "'");
+    // Set up database specific options
+    dbOptions.prefsSafesViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
+    dbOptions.gpiiKeysViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
+    dbOptions.parsedCouchDbUrl = url.parse(dbOptions.couchDbUrl);
+    dbOptions.snapsetPrefsSafes = [];
+    dbOptions.gpiiKeys = [];
+    dbOptions.postOptions = {
+        hostname: dbOptions.parsedCouchDbUrl.hostname,
+        port: dbOptions.parsedCouchDbUrl.port,
+        path: "/gpii/_bulk_docs",
+        method: "POST",
+        headers: {
+            "Accept": "application/json",
+            "Content-Length": 0,          // IMPORTANT: FILL IN PER REQUEST
+            "Content-Type": "application/json"
+        }
+    };
+
+    // Set up function to get at these options.
+    gpii.dataLoader.getOptions = function () {
+        return dbOptions;
+    };
+    fluid.log("COUCHDB_URL: '" +
+        dbOptions.parsedCouchDbUrl.protocol + "//" +
+        dbOptions.parsedCouchDbUrl.hostname + ":" +
+        dbOptions.parsedCouchDbUrl.port +
+        dbOptions.parsedCouchDbUrl.pathname + "'"
+    );
+    fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
+    fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
+};
+dbLoader.initOptions(process.argv);
 
 /**
  * Find the Prefs Safes of type "snapset", mark them to be deleted and add
@@ -69,13 +97,14 @@ fluid.log("BUILD_DATA_DIR: '" + dbLoader.buildDataDir + "'");
  */
 dbLoader.processSnapsets = function (responseString) {
     fluid.log("Processing the snapset Prefs Safes records...");
-    dbLoader.snapSets = JSON.parse(responseString);
-    fluid.each(dbLoader.snapSets.rows, function (aSnapset) {
+    var options = dbLoader.getOptions();
+    var snapSetRecords = JSON.parse(responseString);
+    fluid.each(snapSetRecords.rows, function (aSnapset) {
         aSnapset.value._deleted = true;
-        dbLoader.snapsetPrefsSafes.push(aSnapset.value);
+        options.snapsetPrefsSafes.push(aSnapset.value);
     });
     fluid.log("\tSnapset Prefs Safes marked for deletion.");
-    return dbLoader.snapsetPrefsSafes;
+    return options.snapsetPrefsSafes;
 };
 
 /**
@@ -87,12 +116,13 @@ dbLoader.processSnapsets = function (responseString) {
  */
 dbLoader.processGpiiKeys = function (responseString) {
     fluid.log("Processing the GPII Keys...");
+    var options = dbLoader.getOptions();
     var gpiiKeyRecords = JSON.parse(responseString);
-    dbLoader.gpiiKeys = dbLoader.markPrefsSafesGpiiKeysForDeletion(
-        gpiiKeyRecords, dbLoader.snapsetPrefsSafes
+    options.gpiiKeys = dbLoader.markPrefsSafesGpiiKeysForDeletion(
+        gpiiKeyRecords, options.snapsetPrefsSafes
     );
     fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
-    return dbLoader.gpiiKeys;
+    return options.gpiiKeys;
 };
 
 /**
@@ -124,11 +154,14 @@ dbLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets)
  * Utility to wrap all the pieces to make a bulk documents deletion request
  * for the snapset Prefs Safes and their associated GPII keys.  Its intended use
  * is the parameter of the appropriate promise.then() call.
+ * @param {Object} batchDeleteResponse - the reponse handler configured for the
+ *                                       batch delete request.
  */
-dbLoader.doBatchDelete = function () {
-    var docsToRemove = dbLoader.snapsetPrefsSafes.concat(dbLoader.gpiiKeys);
+dbLoader.doBatchDelete = function (batchDeleteResponse) {
+    var options = dbLoader.getOptions();
+    var docsToRemove = options.snapsetPrefsSafes.concat(options.gpiiKeys);
     var execBatchDelete = dbLoader.createBulkDocsRequest(
-        docsToRemove, dbLoader.batchDeleteResponse
+        docsToRemove, batchDeleteResponse, options
     );
     execBatchDelete();
 };
@@ -137,25 +170,15 @@ dbLoader.doBatchDelete = function () {
  * Create a function that makes a bulk docs POST request using the given data.
  * @param {Object} dataToPost - JSON data to POST and process in bulk.
  * @param {Object} responseHandler - http response handler for the request.
+ * @param {Object} options - Data loader options, specifically the POST options.
  * @return {Function} - A function that wraps an http request to execute the
  *                      POST.
  */
-dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler) {
+dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options) {
     return function () {
-        var postOptions = {
-            hostname: dbLoader.parsedCouchDbUrl.hostname,
-            port: dbLoader.parsedCouchDbUrl.port,
-            path: "/gpii/_bulk_docs",
-            method: "POST",
-            headers: {
-                "Accept": "application/json",
-                "Content-Length": 0,          // filled in below
-                "Content-Type": "application/json"
-            }
-        };
         var batchPostData = JSON.stringify({"docs": dataToPost});
-        postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
-        var batchDocsRequest = http.request(postOptions, responseHandler);
+        options.postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+        var batchDocsRequest = http.request(options.postOptions, responseHandler);
         batchDocsRequest.write(batchPostData);
         batchDocsRequest.end();
         return batchDocsRequest;
@@ -215,7 +238,7 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
  * @param {String} errorMsg - The reason why database access failed.
  */
 dbLoader.bail = function (errorMsg) {
-    fluid.log (errorMsg);
+    fluid.log(errorMsg);
     process.exit(1);
 };
 
@@ -256,72 +279,149 @@ dbLoader.getDataFromDirectory = function (dataDir) {
     return contentArray;
 };
 
-// Load the static data first.  If the database is fresh, there won't be any
-// views to use to find the snapset Prefs Safes.  This ensures they are loaded
-// regardless.  If the data base already has these views, then this is
-// essentially a no-op.
-var staticData = dbLoader.getDataFromDirectory(dbLoader.staticDataDir);
-var staticDataPromise = fluid.promise();
-var staticDataResponse = dbLoader.createResponseHandler(
-    function () {
-        fluid.log("Loading static data from '" + dbLoader.staticDataDir + "'");
-    },
-    staticDataPromise
-);
-var execStaticDataRequest = dbLoader.createBulkDocsRequest(staticData, staticDataResponse);
-execStaticDataRequest();
+/*
+ * Create the step that loads the static data. This should be done first since
+ * if the database is fresh, there won't be any views to use to find the snapset
+ * Prefs Safes.  This ensures the views are loaded regardless.  If the database
+ * already has these views, then this is a no-op.
+ * @param {Object} options - Object that has the path to the directory
+ *                           containing the static data.
+ * @return {Object} - An object containing the database request to load the
+ *                    static data, and a promise configure to trigger the next
+ *                    step.
+ */
+dbLoader.createStaticDataStep = function (options) {
+    var togo = fluid.promise();
+    var data = dbLoader.getDataFromDirectory(options.staticDataDir);
+    var response = dbLoader.createResponseHandler(
+        function () {
+            fluid.log("Loading static data from '" + options.staticDataDir + "'");
+        },
+        togo
+    );
+    var staticDataRequest = dbLoader.createBulkDocsRequest(data, response, options);
+    return { request: staticDataRequest, promise: togo };
+};
 
-// Secondly, get the snapsets Prefs Safes.
-var snapsetsPromise = fluid.promise();
-var getSnapSetsResponse = dbLoader.createResponseHandler(
-    dbLoader.processSnapsets,
-    snapsetsPromise,
-    "Error retrieving snapsets Prefs Safes: "
-);
-var snapSetsRequest = dbLoader.queryDatabase(
-    dbLoader.prefsSafesViewUrl,
-    getSnapSetsResponse,
-    "Error requesting snapsets Prefs Safes: "
-);
-staticDataPromise.then(function () { snapSetsRequest.end(); }, dbLoader.bail);
+/*
+ * Create the step that fetches the current snapset Prefs Safes from the
+ * database.
+ * @param {Object} options - Object that has the view for finding snap set
+ *                           Prefs Safes records in the database.
+ * @param {Promise} previousStep - Promise from a previous step whose fulfillment
+ *                                 triggers the request configured herein.
+ * @return {Promise} - The promise asoociated with this step.
+ */
+dbLoader.createFetchSnapsetsStep = function (options, previousStep) {
+    var togo = fluid.promise();
+    var response = dbLoader.createResponseHandler(
+        dbLoader.processSnapsets,
+        togo,
+        "Error retrieving snapsets Prefs Safes: "
+    );
+    var request = dbLoader.queryDatabase(
+        options.prefsSafesViewUrl,
+        response,
+        "Error requesting snapsets Prefs Safes: "
+    );
+    previousStep.then(function () { request.end(); }, dbLoader.bail);
+    return togo;
+};
 
-// Thirdly, the associated GPII Keys.
-var gpiiKeysPromise = fluid.promise();
-var getGpiiKeysResponse = dbLoader.createResponseHandler(
-    dbLoader.processGpiiKeys,
-    gpiiKeysPromise,
-    "Error finding snapset Prefs Safes associated GPII Keys: "
-);
-var getGpiiKeysRequest = dbLoader.queryDatabase(
-    dbLoader.gpiiKeysViewUrl,
-    getGpiiKeysResponse,
-    "Error requesting GPII Keys: "
-);
-snapsetsPromise.then(function () { getGpiiKeysRequest.end(); }, dbLoader.bail);
+/*
+ * Create the step that fetches the current GPII keys associate with the snapset
+ * Prefs Safes.
+ * @param {Object} options - Object that has the view for finding all GPII Key
+ *                           records in the database.
+ * @param {Promise} previousStep - Promise from a previous step whose fulfillment
+ *                                 triggers the request configured herein.
+ * @return {Promise} - The promise asoociated with this step.
+ */
+dbLoader.createFetchGpiiKeysStep = function (options, previousStep) {
+    var togo = fluid.promise();
+    var response = dbLoader.createResponseHandler(
+        dbLoader.processGpiiKeys,
+        togo,
+        "Error finding snapset Prefs Safes associated GPII Keys: "
+    );
+    var request = dbLoader.queryDatabase(
+        options.gpiiKeysViewUrl,
+        response,
+        "Error requesting GPII Keys: "
+    );
+    previousStep.then(function () { request.end(); }, dbLoader.bail);
+    return togo;
+};
 
-// Next, delete the snapset Prefs Safes and their GPII Keys in batch.
-var batchDeletePromise = fluid.promise();
-dbLoader.batchDeleteResponse = dbLoader.createResponseHandler(
-    function () { fluid.log("Snapset Prefs Safes and associated GPII Keys deleted."); },
-    batchDeletePromise
-);
-gpiiKeysPromise.then(dbLoader.doBatchDelete, dbLoader.bail);
+/*
+ * Create the step that deletes, in batch, the current snapset Prefs Safes and
+ * their associated GPII keys.
+ * @param {Promise} previousStep - Promise from a previous step whose fulfillment
+ *                                 triggers the bulk delete request.
+ * @return {Promise} - The promise asoociated with this step.
+ */
+dbLoader.createBatchDeleteStep = function (previousStep) {
+    var togo = fluid.promise();
+    var response = dbLoader.createResponseHandler(
+        function () {
+            fluid.log("Snapset Prefs Safes and associated GPII Keys deleted.");
+        },
+        togo
+    );
+    previousStep.then(
+        function () { dbLoader.doBatchDelete(response); },
+        dbLoader.bail
+    );
+    return togo;
+};
 
-if (dbLoader.justDelete) {
-    batchDeletePromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
-} else {
-    // Finally, load the latest snapsets data from the build data.
-    var buildData = dbLoader.getDataFromDirectory(dbLoader.buildDataDir);
-    var demoUserData = dbLoader.getDataFromDirectory(dbLoader.demoUserDir);
-    buildData = buildData.concat(demoUserData);
-    var buildDataPromise = fluid.promise();
-    var buildDataResponse = dbLoader.createResponseHandler(
+/*
+ * Create the step that uploads, in batch, the new snapset Prefs Safes, their
+ * associated GPII keys, and the demo user Prefs Safes/GPII Keys.
+ * @param {Object} options - Object that has the paths to the directories that
+ *                           contain the new snapsets and demo user preferences.
+ * @param {Promise} previousStep - Promise from a previous step whose fulfillment
+ *                                 triggers this bulk upload request.
+ * @return {Promise} - The promise asoociated with this step.
+ */
+dbLoader.createBatchUploadStep = function (options, previousStep) {
+    var togo = fluid.promise();
+    var buildData = dbLoader.getDataFromDirectory(options.buildDataDir);
+    var demoUserData = dbLoader.getDataFromDirectory(options.demoUserDir);
+    var allData = buildData.concat(demoUserData);
+    var response = dbLoader.createResponseHandler(
         function () {
-            fluid.log ("Bulk loading of build data from '" + dbLoader.buildDataDir + "'");
+            fluid.log ("Bulk loading of build data from '" + options.buildDataDir + "'");
         },
-        buildDataPromise
+        togo
     );
-    var execBuildDataRequest = dbLoader.createBulkDocsRequest(buildData, buildDataResponse);
-    batchDeletePromise.then(execBuildDataRequest, dbLoader.bail);
-    buildDataPromise.then(function () { fluid.log("Done."); }, dbLoader.bail);
-}
+    var request = dbLoader.createBulkDocsRequest(allData, response, options);
+    previousStep.then(request, dbLoader.bail);
+    return togo;
+};
+
+/*
+ * Create the steps to load the static data, find and delete the current snapset
+ * Prefs Safes and the GPII keys, and then load the latest snapset Prefs Safes
+ * and their Keys, and the demo user Prefs Safes and their GPII Keys.  After all
+ * steps are configured and connected, trigger the first one.
+ */
+dbLoader.orchestrate = function () {
+    var options = dbLoader.getOptions();
+    var lastStep;
+    var firstStep = dbLoader.createStaticDataStep(options);
+    var aStep = dbLoader.createFetchSnapsetsStep(options, firstStep.promise);
+    aStep = dbLoader.createFetchGpiiKeysStep(options, aStep);
+    aStep = dbLoader.createBatchDeleteStep(aStep);
+    if (options.justDelete) {
+        lastStep = aStep;
+    } else {
+        lastStep = dbLoader.createBatchUploadStep(options, aStep);
+    };
+    // Go!
+    firstStep.request();
+    lastStep.then(function () { fluid.log("Done."); }, dbLoader.bail);
+};
+
+dbLoader.orchestrate();
+

From c401c9749d5a75299a71795eea3a0229410f64b1 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 7 Aug 2018 12:30:06 -0400
Subject: [PATCH 25/41] GPII-3138:  Update snapsets in the database

Modified the response handler for the static data step to provide
more info about the result to help debug why the views are not
uploaded when running in AWS.
---
 scripts/deleteAndLoadSnapsets.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 4c1100c3e..2e270605e 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -294,8 +294,9 @@ dbLoader.createStaticDataStep = function (options) {
     var togo = fluid.promise();
     var data = dbLoader.getDataFromDirectory(options.staticDataDir);
     var response = dbLoader.createResponseHandler(
-        function () {
+        function (responseString) {
             fluid.log("Loading static data from '" + options.staticDataDir + "'");
+            fluid.log("\tresponse: '" + responseString + "'");
         },
         togo
     );

From 54defe755ec44e6aac46cc104cfaef215b0bc936 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 7 Aug 2018 16:06:10 -0400
Subject: [PATCH 26/41] GPII-3138:  Update snapsets in the database

Modified to no longer load the static data.  That is now handled
by the gpii-dataloader bash script prior to its calling this node
script.
---
 scripts/deleteAndLoadSnapsets.js | 74 ++++++++++----------------------
 1 file changed, 22 insertions(+), 52 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 2e270605e..6af58330a 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -9,14 +9,13 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 */
 
 // This script modifies the preferences database:
-// 1. Loads static data into the database,
-// 2. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
-// 3. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
-// 4. Deletes these Prefs Safes and their associated GPII Keys from the database,
-// 5. Uploads the new Prefs Safes and their GPII Keys to the database,
-// 6. Uploads demo user Prefs Safes and their Keys in to the database, if they are not already present.
+// 1. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
+// 2. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
+// 3. Deletes these Prefs Safes and their associated GPII Keys from the database,
+// 4. Uploads the new Prefs Safes and their GPII Keys to the database,
+// 5. Uploads demo user Prefs Safes and their Keys in to the database, if they are not already present.
 // A sample command that runs this script:
-// node deleteAndLoadSnapsets.js $COUCHDBURL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR
+// node deleteAndLoadSnapsets.js $COUCHDBURL $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR
 
 "use strict";
 
@@ -31,7 +30,7 @@ fluid.setLogging(fluid.logLevel.INFO);
 
 // Handle command line
 if (process.argv.length < 5) {
-    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
     process.exit(1);
 }
 
@@ -46,10 +45,9 @@ var dbLoader = gpii.dataLoader;
 dbLoader.initOptions = function (processArgv) {
     var dbOptions = {};
     dbOptions.couchDbUrl = processArgv[2];
-    dbOptions.staticDataDir = processArgv[3];
-    dbOptions.buildDataDir = processArgv[4];
-    dbOptions.demoUserDir = processArgv[5];
-    if (processArgv.length > 6 && processArgv[6] === "--justDelete") { // for debugging.
+    dbOptions.buildDataDir = processArgv[3];
+    dbOptions.demoUserDir = processArgv[4];
+    if (processArgv.length > 5 && processArgv[5] === "--justDelete") { // for debugging.
         dbOptions.justDelete = true;
     } else {
         dbOptions.justDelete = false;
@@ -83,7 +81,6 @@ dbLoader.initOptions = function (processArgv) {
         dbOptions.parsedCouchDbUrl.port +
         dbOptions.parsedCouchDbUrl.pathname + "'"
     );
-    fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
 };
 dbLoader.initOptions(process.argv);
@@ -279,54 +276,28 @@ dbLoader.getDataFromDirectory = function (dataDir) {
     return contentArray;
 };
 
-/*
- * Create the step that loads the static data. This should be done first since
- * if the database is fresh, there won't be any views to use to find the snapset
- * Prefs Safes.  This ensures the views are loaded regardless.  If the database
- * already has these views, then this is a no-op.
- * @param {Object} options - Object that has the path to the directory
- *                           containing the static data.
- * @return {Object} - An object containing the database request to load the
- *                    static data, and a promise configure to trigger the next
- *                    step.
- */
-dbLoader.createStaticDataStep = function (options) {
-    var togo = fluid.promise();
-    var data = dbLoader.getDataFromDirectory(options.staticDataDir);
-    var response = dbLoader.createResponseHandler(
-        function (responseString) {
-            fluid.log("Loading static data from '" + options.staticDataDir + "'");
-            fluid.log("\tresponse: '" + responseString + "'");
-        },
-        togo
-    );
-    var staticDataRequest = dbLoader.createBulkDocsRequest(data, response, options);
-    return { request: staticDataRequest, promise: togo };
-};
-
 /*
  * Create the step that fetches the current snapset Prefs Safes from the
  * database.
  * @param {Object} options - Object that has the view for finding snap set
  *                           Prefs Safes records in the database.
- * @param {Promise} previousStep - Promise from a previous step whose fulfillment
- *                                 triggers the request configured herein.
- * @return {Promise} - The promise asoociated with this step.
+ * @return {Object} - An object containing the database request to load the
+ *                    static data, and a promise configure to trigger the next
+ *                    step.
  */
-dbLoader.createFetchSnapsetsStep = function (options, previousStep) {
+dbLoader.createFetchSnapsetsStep = function (options) {
     var togo = fluid.promise();
     var response = dbLoader.createResponseHandler(
         dbLoader.processSnapsets,
         togo,
         "Error retrieving snapsets Prefs Safes: "
     );
-    var request = dbLoader.queryDatabase(
+    var snapsSetsRequest = dbLoader.queryDatabase(
         options.prefsSafesViewUrl,
         response,
         "Error requesting snapsets Prefs Safes: "
     );
-    previousStep.then(function () { request.end(); }, dbLoader.bail);
-    return togo;
+    return { request: snapsSetsRequest, promise: togo };
 };
 
 /*
@@ -410,17 +381,16 @@ dbLoader.createBatchUploadStep = function (options, previousStep) {
 dbLoader.orchestrate = function () {
     var options = dbLoader.getOptions();
     var lastStep;
-    var firstStep = dbLoader.createStaticDataStep(options);
-    var aStep = dbLoader.createFetchSnapsetsStep(options, firstStep.promise);
-    aStep = dbLoader.createFetchGpiiKeysStep(options, aStep);
-    aStep = dbLoader.createBatchDeleteStep(aStep);
+    var firstStep = dbLoader.createFetchSnapsetsStep(options);
+    var nextStep = dbLoader.createFetchGpiiKeysStep(options, firstStep.promise);
+    nextStep = dbLoader.createBatchDeleteStep(nextStep);
     if (options.justDelete) {
-        lastStep = aStep;
+        lastStep = nextStep;
     } else {
-        lastStep = dbLoader.createBatchUploadStep(options, aStep);
+        lastStep = dbLoader.createBatchUploadStep(options, nextStep);
     };
     // Go!
-    firstStep.request();
+    firstStep.request.end();
     lastStep.then(function () { fluid.log("Done."); }, dbLoader.bail);
 };
 

From 58fdde755bd15a4f3ff4b5580632ae26600d46ae Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Thu, 9 Aug 2018 15:50:43 -0400
Subject: [PATCH 27/41] GPII-3138:  Update snapsets in the database

Added Prefs Safes type to command line example (e.g., "snapset").
---
 scripts/convertPrefs.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/convertPrefs.js b/scripts/convertPrefs.js
index 8fec59016..4a6892e2f 100644
--- a/scripts/convertPrefs.js
+++ b/scripts/convertPrefs.js
@@ -14,7 +14,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 // Usage: node scripts/convertPrefs.js {input_path} {target_path}
 //
 // A sample command that runs this script in the universal root directory:
-// node scripts/convertPrefs.js testData/preferences/ build/dbData/
+// node scripts/convertPrefs.js testData/preferences/ build/dbData/ snapset
 
 "use strict";
 

From 2b8cbebabf43f6bd9c41f82a59a603b18ae25a5f Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 28 Aug 2018 16:09:53 -0400
Subject: [PATCH 28/41] GPII-3138:  Update snapsets in the database

Added structure to the empty demo user prefs as per Javi's
comment/request.
---
 .../20558561-dc5b-4973-85f3-9bca975090cb.json5        | 11 ++++++++++-
 .../226722c6-8895-40b4-9297-424b9da9c340.json5        | 11 ++++++++++-
 .../2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5        | 11 ++++++++++-
 .../2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5        | 11 ++++++++++-
 .../2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5        | 11 ++++++++++-
 .../339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5        | 11 ++++++++++-
 .../587ad68c-6470-486b-9828-6ee224ae6902.json5        | 11 ++++++++++-
 .../7009c709-9ce0-441f-9b56-aec7827a1d5c.json5        | 11 ++++++++++-
 .../80489c34-50cd-43db-a1bc-9a50960d5b3a.json5        | 11 ++++++++++-
 .../8060c500-6be9-462a-8ae1-8705be497916.json5        | 11 ++++++++++-
 .../8070295c-d4dc-4b02-9113-ac08f5b0484a.json5        | 11 ++++++++++-
 .../881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5        | 11 ++++++++++-
 .../a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5        | 11 ++++++++++-
 .../b7a438ab-cf2a-472d-9a96-5d87ee187770.json5        | 11 ++++++++++-
 .../c3657704-04f6-4f02-a777-c361f0110876.json5        | 11 ++++++++++-
 .../cd5ca821-6a6e-4734-b956-83546bae1a5f.json5        | 11 ++++++++++-
 .../cead392d-05da-4ecc-9bd3-88c789f94d8c.json5        | 11 ++++++++++-
 .../dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5        | 11 ++++++++++-
 .../e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5        | 11 ++++++++++-
 .../e8c62569-ac93-4c45-8085-f172ca570311.json5        | 11 ++++++++++-
 20 files changed, 200 insertions(+), 20 deletions(-)

diff --git a/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5 b/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
+++ b/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5 b/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
+++ b/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5 b/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
+++ b/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5 b/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
+++ b/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5 b/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
+++ b/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5 b/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
+++ b/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5 b/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
+++ b/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5 b/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
+++ b/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5 b/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
+++ b/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5 b/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
+++ b/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5 b/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
+++ b/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5 b/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
+++ b/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5 b/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
+++ b/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5 b/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
+++ b/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5 b/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
+++ b/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5 b/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
+++ b/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5 b/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
+++ b/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5 b/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
+++ b/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5 b/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
+++ b/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}
diff --git a/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5 b/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
index 9f7a319dd..98cd9afad 100644
--- a/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
+++ b/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
@@ -1,2 +1,11 @@
 // Empty user key
-{}
+{
+    "flat": {
+        "contexts": {
+            "gpii-default": {
+                "name": "Default preferences",
+                "preferences": {}
+            }
+        }
+    }
+}

From 59f03a8f74757f4bcfd90e5246ecf8d357bd55e7 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Thu, 30 Aug 2018 15:56:56 -0400
Subject: [PATCH 29/41] GPII-3138:  Update snapsets in the database

- removed accessor function for options, passing them everywhere as
necessary
- use full name space for the dataloader object.
---
 scripts/deleteAndLoadSnapsets.js | 153 ++++++++++++++++++-------------
 1 file changed, 88 insertions(+), 65 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 6af58330a..e90a965c7 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -26,6 +26,7 @@ var http = require("http"),
 
 var gpii = fluid.registerNamespace("gpii");
 fluid.registerNamespace("gpii.dataLoader");
+
 fluid.setLogging(fluid.logLevel.INFO);
 
 // Handle command line
@@ -34,15 +35,14 @@ if (process.argv.length < 5) {
     process.exit(1);
 }
 
-var dbLoader = gpii.dataLoader;
-
 /**
  * Create a set of options for data loader and a function to retreive them.
  * The options are based on the command line parameters and a set of database
  * constants.
  * @param {Array} processArgv - The command line arguments.
+ * @return {Object} - The options.
  */
-dbLoader.initOptions = function (processArgv) {
+gpii.dataLoader.initOptions = function (processArgv) {
     var dbOptions = {};
     dbOptions.couchDbUrl = processArgv[2];
     dbOptions.buildDataDir = processArgv[3];
@@ -70,11 +70,6 @@ dbLoader.initOptions = function (processArgv) {
             "Content-Type": "application/json"
         }
     };
-
-    // Set up function to get at these options.
-    gpii.dataLoader.getOptions = function () {
-        return dbOptions;
-    };
     fluid.log("COUCHDB_URL: '" +
         dbOptions.parsedCouchDbUrl.protocol + "//" +
         dbOptions.parsedCouchDbUrl.hostname + ":" +
@@ -82,19 +77,20 @@ dbLoader.initOptions = function (processArgv) {
         dbOptions.parsedCouchDbUrl.pathname + "'"
     );
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
+    return dbOptions;
 };
-dbLoader.initOptions(process.argv);
 
 /**
  * Find the Prefs Safes of type "snapset", mark them to be deleted and add
  * them to an array of records to remove.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving the snapset PrefsSafes records
+ * @param {Object} options - Data loader options used to store the current set
+ *                           of processed snapsets.
  * @return {Array} - The snapset PrefsSafes records marked for deletion.
  */
-dbLoader.processSnapsets = function (responseString) {
+gpii.dataLoader.processSnapsets = function (responseString, options) {
     fluid.log("Processing the snapset Prefs Safes records...");
-    var options = dbLoader.getOptions();
     var snapSetRecords = JSON.parse(responseString);
     fluid.each(snapSetRecords.rows, function (aSnapset) {
         aSnapset.value._deleted = true;
@@ -109,13 +105,14 @@ dbLoader.processSnapsets = function (responseString) {
  * them for deletion, and add them to array of records to delete.
  * @param {String} responseString - The response from the database query for
  *                                  retrieving all the GPII Keys.
+ * @param {Object} options - Data loader options used to store the current set
+ *                           of snapset GPII keys.
  * @return {Array} - The GPII Key records marked for deletion.
  */
-dbLoader.processGpiiKeys = function (responseString) {
+gpii.dataLoader.processGpiiKeys = function (responseString, options) {
     fluid.log("Processing the GPII Keys...");
-    var options = dbLoader.getOptions();
     var gpiiKeyRecords = JSON.parse(responseString);
-    options.gpiiKeys = dbLoader.markPrefsSafesGpiiKeysForDeletion(
+    options.gpiiKeys = gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion(
         gpiiKeyRecords, options.snapsetPrefsSafes
     );
     fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
@@ -131,7 +128,7 @@ dbLoader.processGpiiKeys = function (responseString) {
  *                           its associated GPII Key record.
  * @return {Array} - the values from the gpiiKeyRecords that are snapset GPII Keys.
  */
-dbLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets) {
+gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets) {
     var gpiiKeysToDelete = [];
     fluid.each(gpiiKeyRecords.rows, function (gpiiKeyRecord) {
         var gpiiKey = fluid.find(snapSets, function (aSnapSet) {
@@ -153,11 +150,11 @@ dbLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets)
  * is the parameter of the appropriate promise.then() call.
  * @param {Object} batchDeleteResponse - the reponse handler configured for the
  *                                       batch delete request.
+ * @param {Object} options - Object that contains the records to be deleted.
  */
-dbLoader.doBatchDelete = function (batchDeleteResponse) {
-    var options = dbLoader.getOptions();
+gpii.dataLoader.doBatchDelete = function (batchDeleteResponse, options) {
     var docsToRemove = options.snapsetPrefsSafes.concat(options.gpiiKeys);
-    var execBatchDelete = dbLoader.createBulkDocsRequest(
+    var execBatchDelete = gpii.dataLoader.createBulkDocsRequest(
         docsToRemove, batchDeleteResponse, options
     );
     execBatchDelete();
@@ -171,7 +168,7 @@ dbLoader.doBatchDelete = function (batchDeleteResponse) {
  * @return {Function} - A function that wraps an http request to execute the
  *                      POST.
  */
-dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options) {
+gpii.dataLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options) {
     return function () {
         var batchPostData = JSON.stringify({"docs": dataToPost});
         options.postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
@@ -187,13 +184,14 @@ dbLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options)
  * at the correct time.
  * @param {Function} handleEnd - Function to call that deals with the response
  *                               data when the response receives an "end" event.
- * @param {Promise} promise - Promose to resolve/reject on a response "end" or
+ * @param {Object} options - Data loader options passed to handleEnd().
+ * @param {Promise} promise - Promise to resolve/reject on a response "end" or
  *                           "error" event.
  * @param {String} errorMsg - Optional error message to prepend to the error
  *                            received from a response "error" event.
  * @return {Function} - Function reponse callback for an http request.
  */
-dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
+gpii.dataLoader.createResponseHandler = function (handleEnd, options, promise, errorMsg) {
     return function (response) {
         var responseString = "";
 
@@ -216,7 +214,7 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
                 promise.reject(fullErrorMsg);
             }
             else {
-                var value = handleEnd(responseString);
+                var value = handleEnd(responseString, options);
                 promise.resolve(value);
             }
         });
@@ -231,10 +229,10 @@ dbLoader.createResponseHandler = function (handleEnd, promise, errorMsg) {
  * Quit the whole process because a request of the database has failed, and log
  * the error.  Use this function when the failure has not actually modified the
  * database; for example, when getting all the current snapset Prefs Safes.  If
- * that failed, that database is as it was, but there is no point in continuing.
+ * that failed, the database is unchanged, but there is no point in continuing.
  * @param {String} errorMsg - The reason why database access failed.
  */
-dbLoader.bail = function (errorMsg) {
+gpii.dataLoader.bail = function (errorMsg) {
     fluid.log(errorMsg);
     process.exit(1);
 };
@@ -249,7 +247,7 @@ dbLoader.bail = function (errorMsg) {
  * @param {String} errorMsg - optional error message for request errors.
  * @return {http.ClientRequest} - The http request object.
  */
-dbLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg) {
+gpii.dataLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg) {
     var aRequest = http.request(databaseURL, handleResponse);
     aRequest.on("error", function (e) {
         fluid.log(errorMsg + e.message);
@@ -264,7 +262,7 @@ dbLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg) {
  * @return {Array} - Each element of the array is an Object based on the
  *                   contents of each file loaded.
  */
-dbLoader.getDataFromDirectory = function (dataDir) {
+gpii.dataLoader.getDataFromDirectory = function (dataDir) {
     var contentArray = [];
     var files = fs.readdirSync(dataDir);
     files.forEach(function (aFile) {
@@ -282,17 +280,18 @@ dbLoader.getDataFromDirectory = function (dataDir) {
  * @param {Object} options - Object that has the view for finding snap set
  *                           Prefs Safes records in the database.
  * @return {Object} - An object containing the database request to load the
- *                    static data, and a promise configure to trigger the next
+ *                    static data, and a promise configured to trigger the next
  *                    step.
  */
-dbLoader.createFetchSnapsetsStep = function (options) {
+gpii.dataLoader.createFetchSnapsetsStep = function (options) {
     var togo = fluid.promise();
-    var response = dbLoader.createResponseHandler(
-        dbLoader.processSnapsets,
+    var response = gpii.dataLoader.createResponseHandler(
+        gpii.dataLoader.processSnapsets,
+        options,
         togo,
         "Error retrieving snapsets Prefs Safes: "
     );
-    var snapsSetsRequest = dbLoader.queryDatabase(
+    var snapsSetsRequest = gpii.dataLoader.queryDatabase(
         options.prefsSafesViewUrl,
         response,
         "Error requesting snapsets Prefs Safes: "
@@ -303,72 +302,97 @@ dbLoader.createFetchSnapsetsStep = function (options) {
 /*
  * Create the step that fetches the current GPII keys associate with the snapset
  * Prefs Safes.
- * @param {Object} options - Object that has the view for finding all GPII Key
- *                           records in the database.
  * @param {Promise} previousStep - Promise from a previous step whose fulfillment
  *                                 triggers the request configured herein.
+ * @param {Object} options - Object that has the view for finding all GPII Key
+ *                           records in the database.
  * @return {Promise} - The promise asoociated with this step.
  */
-dbLoader.createFetchGpiiKeysStep = function (options, previousStep) {
+gpii.dataLoader.createFetchGpiiKeysStep = function (previousStep, options) {
     var togo = fluid.promise();
-    var response = dbLoader.createResponseHandler(
-        dbLoader.processGpiiKeys,
+    var response = gpii.dataLoader.createResponseHandler(
+        gpii.dataLoader.processGpiiKeys,
+        options,
         togo,
         "Error finding snapset Prefs Safes associated GPII Keys: "
     );
-    var request = dbLoader.queryDatabase(
+    var request = gpii.dataLoader.queryDatabase(
         options.gpiiKeysViewUrl,
         response,
         "Error requesting GPII Keys: "
     );
-    previousStep.then(function () { request.end(); }, dbLoader.bail);
+    previousStep.then(function () { request.end(); }, gpii.dataLoader.gbail);
     return togo;
 };
 
+/*
+ * Log how many snapset Prefs Safes and GPII Keys were deleted.
+ * @param {String} responseString - Response from the database
+ * @param {Object} options - Object that contains the sets of Prefs Safes and
+ *                           their keys.
+ */
+gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
+    fluid.log(  "Deleted " +
+                options.snapsetPrefsSafes.length + " Prefs Safes and " +
+                options.gpiiKeys.length + " associated GPII Keys, " +
+                "response:  " + responseString
+    );
+};
+
 /*
  * Create the step that deletes, in batch, the current snapset Prefs Safes and
  * their associated GPII keys.
  * @param {Promise} previousStep - Promise from a previous step whose fulfillment
  *                                 triggers the bulk delete request.
+ * @param {Object} options - Object that contains the records to be deleted.
  * @return {Promise} - The promise asoociated with this step.
  */
-dbLoader.createBatchDeleteStep = function (previousStep) {
+gpii.dataLoader.createBatchDeleteStep = function (previousStep, options) {
     var togo = fluid.promise();
-    var response = dbLoader.createResponseHandler(
-        function () {
-            fluid.log("Snapset Prefs Safes and associated GPII Keys deleted.");
-        },
+    var response = gpii.dataLoader.createResponseHandler(
+        gpii.dataLoader.logSnapsetDeletion,
+        options,
         togo
     );
     previousStep.then(
-        function () { dbLoader.doBatchDelete(response); },
-        dbLoader.bail
+        function () { gpii.dataLoader.doBatchDelete(response, options); },
+        gpii.dataLoader.bail
     );
     return togo;
 };
 
+/*
+ * Log the uploading of all the snapset and user Prefs Safes.
+ * @param {String} responseString - Response from the database (ignored)
+ * @param {Object} options - Object that contains the directories from which
+ *                           the Prefs Safes were loaded.
+ */
+gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
+    fluid.log ("Bulk loading of build data from '" + options.buildDataDir + "'");
+    fluid.log ("Bulk loading of demo user data from '" + options.demoUserDir + "'");
+};
+
 /*
  * Create the step that uploads, in batch, the new snapset Prefs Safes, their
  * associated GPII keys, and the demo user Prefs Safes/GPII Keys.
- * @param {Object} options - Object that has the paths to the directories that
- *                           contain the new snapsets and demo user preferences.
  * @param {Promise} previousStep - Promise from a previous step whose fulfillment
  *                                 triggers this bulk upload request.
+ * @param {Object} options - Object that has the paths to the directories that
+ *                           contain the new snapsets and demo user preferences.
  * @return {Promise} - The promise asoociated with this step.
  */
-dbLoader.createBatchUploadStep = function (options, previousStep) {
+gpii.dataLoader.createBatchUploadStep = function (previousStep, options) {
     var togo = fluid.promise();
-    var buildData = dbLoader.getDataFromDirectory(options.buildDataDir);
-    var demoUserData = dbLoader.getDataFromDirectory(options.demoUserDir);
+    var buildData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
+    var demoUserData = gpii.dataLoader.getDataFromDirectory(options.demoUserDir);
     var allData = buildData.concat(demoUserData);
-    var response = dbLoader.createResponseHandler(
-        function () {
-            fluid.log ("Bulk loading of build data from '" + options.buildDataDir + "'");
-        },
+    var response = gpii.dataLoader.createResponseHandler(
+        gpii.dataLoader.logSnapsetsUpload,
+        options,
         togo
     );
-    var request = dbLoader.createBulkDocsRequest(allData, response, options);
-    previousStep.then(request, dbLoader.bail);
+    var request = gpii.dataLoader.createBulkDocsRequest(allData, response, options);
+    previousStep.then(request, gpii.dataLoader.bail);
     return togo;
 };
 
@@ -378,21 +402,20 @@ dbLoader.createBatchUploadStep = function (options, previousStep) {
  * and their Keys, and the demo user Prefs Safes and their GPII Keys.  After all
  * steps are configured and connected, trigger the first one.
  */
-dbLoader.orchestrate = function () {
-    var options = dbLoader.getOptions();
+gpii.dataLoader.orchestrate = function () {
+    var options = gpii.dataLoader.initOptions(process.argv);
     var lastStep;
-    var firstStep = dbLoader.createFetchSnapsetsStep(options);
-    var nextStep = dbLoader.createFetchGpiiKeysStep(options, firstStep.promise);
-    nextStep = dbLoader.createBatchDeleteStep(nextStep);
+    var firstStep = gpii.dataLoader.createFetchSnapsetsStep(options);
+    var nextStep = gpii.dataLoader.createFetchGpiiKeysStep(firstStep.promise, options);
+    nextStep = gpii.dataLoader.createBatchDeleteStep(nextStep, options);
     if (options.justDelete) {
         lastStep = nextStep;
     } else {
-        lastStep = dbLoader.createBatchUploadStep(options, nextStep);
+        lastStep = gpii.dataLoader.createBatchUploadStep(nextStep, options);
     };
     // Go!
     firstStep.request.end();
-    lastStep.then(function () { fluid.log("Done."); }, dbLoader.bail);
+    lastStep.then(function () { fluid.log("Done."); }, gpii.dataLoader.bail);
 };
 
-dbLoader.orchestrate();
-
+gpii.dataLoader.orchestrate();

From 6ee716e5db751246d3b04eca8beee91b491f5473 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 10 Sep 2018 10:38:35 -0400
Subject: [PATCH 30/41] GPII-3138:  Update snapsets in the database

Using fluid.promise.sequence() to orchestrate the steps.
---
 scripts/deleteAndLoadSnapsets.js | 124 ++++++++++++++-----------------
 1 file changed, 55 insertions(+), 69 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index e90a965c7..058d82e69 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -151,13 +151,13 @@ gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, sn
  * @param {Object} batchDeleteResponse - the reponse handler configured for the
  *                                       batch delete request.
  * @param {Object} options - Object that contains the records to be deleted.
+ * @return {http.ClientRequest} - The http request object.
  */
-gpii.dataLoader.doBatchDelete = function (batchDeleteResponse, options) {
+gpii.dataLoader.configureBatchDelete = function (batchDeleteResponse, options) {
     var docsToRemove = options.snapsetPrefsSafes.concat(options.gpiiKeys);
-    var execBatchDelete = gpii.dataLoader.createBulkDocsRequest(
+    return gpii.dataLoader.createBulkDocsRequest(
         docsToRemove, batchDeleteResponse, options
     );
-    execBatchDelete();
 };
 
 /**
@@ -165,18 +165,14 @@ gpii.dataLoader.doBatchDelete = function (batchDeleteResponse, options) {
  * @param {Object} dataToPost - JSON data to POST and process in bulk.
  * @param {Object} responseHandler - http response handler for the request.
  * @param {Object} options - Data loader options, specifically the POST options.
- * @return {Function} - A function that wraps an http request to execute the
- *                      POST.
+ * @return {http.ClientRequest} - An http request object.
  */
 gpii.dataLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options) {
-    return function () {
-        var batchPostData = JSON.stringify({"docs": dataToPost});
-        options.postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
-        var batchDocsRequest = http.request(options.postOptions, responseHandler);
-        batchDocsRequest.write(batchPostData);
-        batchDocsRequest.end();
-        return batchDocsRequest;
-    };
+    var batchPostData = JSON.stringify({"docs": dataToPost});
+    options.postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+    var batchDocsRequest = http.request(options.postOptions, responseHandler);
+    batchDocsRequest.write(batchPostData);
+    return batchDocsRequest;
 };
 
 /**
@@ -225,18 +221,6 @@ gpii.dataLoader.createResponseHandler = function (handleEnd, options, promise, e
     };
 };
 
-/**
- * Quit the whole process because a request of the database has failed, and log
- * the error.  Use this function when the failure has not actually modified the
- * database; for example, when getting all the current snapset Prefs Safes.  If
- * that failed, the database is unchanged, but there is no point in continuing.
- * @param {String} errorMsg - The reason why database access failed.
- */
-gpii.dataLoader.bail = function (errorMsg) {
-    fluid.log(errorMsg);
-    process.exit(1);
-};
-
 /**
  * General mechanism to create a database request, set up an error handler and
  * return.  It is up to the caller to trigger the request by calling its end()
@@ -279,9 +263,8 @@ gpii.dataLoader.getDataFromDirectory = function (dataDir) {
  * database.
  * @param {Object} options - Object that has the view for finding snap set
  *                           Prefs Safes records in the database.
- * @return {Object} - An object containing the database request to load the
- *                    static data, and a promise configured to trigger the next
- *                    step.
+ * @return {Promise} - A promise that resolves to the set of snapset PrefsSafes
+ *                     currently in the database.
  */
 gpii.dataLoader.createFetchSnapsetsStep = function (options) {
     var togo = fluid.promise();
@@ -291,24 +274,24 @@ gpii.dataLoader.createFetchSnapsetsStep = function (options) {
         togo,
         "Error retrieving snapsets Prefs Safes: "
     );
-    var snapsSetsRequest = gpii.dataLoader.queryDatabase(
+    var snapSetsRequest = gpii.dataLoader.queryDatabase(
         options.prefsSafesViewUrl,
         response,
         "Error requesting snapsets Prefs Safes: "
     );
-    return { request: snapsSetsRequest, promise: togo };
+    snapSetsRequest.end();
+    return togo;
 };
 
 /*
  * Create the step that fetches the current GPII keys associate with the snapset
  * Prefs Safes.
- * @param {Promise} previousStep - Promise from a previous step whose fulfillment
- *                                 triggers the request configured herein.
  * @param {Object} options - Object that has the view for finding all GPII Key
  *                           records in the database.
- * @return {Promise} - The promise asoociated with this step.
+ * @return {Promise} - A promise that resolves to the set of GPII keys in the
+ *                     database that correspond to snapset PrefsSafes.
  */
-gpii.dataLoader.createFetchGpiiKeysStep = function (previousStep, options) {
+gpii.dataLoader.createFetchGpiiKeysStep = function (options) {
     var togo = fluid.promise();
     var response = gpii.dataLoader.createResponseHandler(
         gpii.dataLoader.processGpiiKeys,
@@ -321,22 +304,26 @@ gpii.dataLoader.createFetchGpiiKeysStep = function (previousStep, options) {
         response,
         "Error requesting GPII Keys: "
     );
-    previousStep.then(function () { request.end(); }, gpii.dataLoader.gbail);
+    request.end();
     return togo;
 };
 
 /*
  * Log how many snapset Prefs Safes and GPII Keys were deleted.
- * @param {String} responseString - Response from the database
+ * @param {String} responseString - Response from the database (ignored)
  * @param {Object} options - Object that contains the sets of Prefs Safes and
  *                           their keys.
+ * @return {Object} - The number of snapsets and gpiiKeys deleted.
  */
 gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
     fluid.log(  "Deleted " +
                 options.snapsetPrefsSafes.length + " Prefs Safes and " +
-                options.gpiiKeys.length + " associated GPII Keys, " +
-                "response:  " + responseString
+                options.gpiiKeys.length + " associated GPII Keys, "
     );
+    return {
+        snapsets: options.snapsetPrefsSafes.length,
+        gpiiKeys: options.gpiiKeys.length
+    };
 };
 
 /*
@@ -345,19 +332,17 @@ gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
  * @param {Promise} previousStep - Promise from a previous step whose fulfillment
  *                                 triggers the bulk delete request.
  * @param {Object} options - Object that contains the records to be deleted.
- * @return {Promise} - The promise asoociated with this step.
+ * @return {Promise} - The promise that resolves the deletion.
  */
-gpii.dataLoader.createBatchDeleteStep = function (previousStep, options) {
+gpii.dataLoader.createBatchDeleteStep = function (options) {
     var togo = fluid.promise();
     var response = gpii.dataLoader.createResponseHandler(
         gpii.dataLoader.logSnapsetDeletion,
         options,
         togo
     );
-    previousStep.then(
-        function () { gpii.dataLoader.doBatchDelete(response, options); },
-        gpii.dataLoader.bail
-    );
+    var batchDeleteRequest = gpii.dataLoader.configureBatchDelete(response, options);
+    batchDeleteRequest.end();
     return togo;
 };
 
@@ -368,20 +353,19 @@ gpii.dataLoader.createBatchDeleteStep = function (previousStep, options) {
  *                           the Prefs Safes were loaded.
  */
 gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
-    fluid.log ("Bulk loading of build data from '" + options.buildDataDir + "'");
-    fluid.log ("Bulk loading of demo user data from '" + options.demoUserDir + "'");
+    fluid.log("Bulk loading of build data from '" + options.buildDataDir + "'");
+    fluid.log("Bulk loading of demo user data from '" + options.demoUserDir + "'");
+    return "Uploaded latest snapsets and demo user preferences";
 };
 
 /*
  * Create the step that uploads, in batch, the new snapset Prefs Safes, their
  * associated GPII keys, and the demo user Prefs Safes/GPII Keys.
- * @param {Promise} previousStep - Promise from a previous step whose fulfillment
- *                                 triggers this bulk upload request.
  * @param {Object} options - Object that has the paths to the directories that
  *                           contain the new snapsets and demo user preferences.
- * @return {Promise} - The promise asoociated with this step.
+ * @return {Promise} - A promise that resolves the upload.
  */
-gpii.dataLoader.createBatchUploadStep = function (previousStep, options) {
+gpii.dataLoader.createBatchUploadStep = function (options) {
     var togo = fluid.promise();
     var buildData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
     var demoUserData = gpii.dataLoader.getDataFromDirectory(options.demoUserDir);
@@ -391,31 +375,33 @@ gpii.dataLoader.createBatchUploadStep = function (previousStep, options) {
         options,
         togo
     );
-    var request = gpii.dataLoader.createBulkDocsRequest(allData, response, options);
-    previousStep.then(request, gpii.dataLoader.bail);
+    var bulkUploadRequest = gpii.dataLoader.createBulkDocsRequest(allData, response, options);
+    bulkUploadRequest.end();
     return togo;
 };
 
 /*
- * Create the steps to load the static data, find and delete the current snapset
- * Prefs Safes and the GPII keys, and then load the latest snapset Prefs Safes
- * and their Keys, and the demo user Prefs Safes and their GPII Keys.  After all
- * steps are configured and connected, trigger the first one.
+ * Create and execute the steps to update the database.
  */
 gpii.dataLoader.orchestrate = function () {
     var options = gpii.dataLoader.initOptions(process.argv);
-    var lastStep;
-    var firstStep = gpii.dataLoader.createFetchSnapsetsStep(options);
-    var nextStep = gpii.dataLoader.createFetchGpiiKeysStep(firstStep.promise, options);
-    nextStep = gpii.dataLoader.createBatchDeleteStep(nextStep, options);
-    if (options.justDelete) {
-        lastStep = nextStep;
-    } else {
-        lastStep = gpii.dataLoader.createBatchUploadStep(nextStep, options);
-    };
-    // Go!
-    firstStep.request.end();
-    lastStep.then(function () { fluid.log("Done."); }, gpii.dataLoader.bail);
+    var sequence = [
+        gpii.dataLoader.createFetchSnapsetsStep,
+        gpii.dataLoader.createFetchGpiiKeysStep,
+        gpii.dataLoader.createBatchDeleteStep
+    ];
+    if (!options.justDelete) {
+        sequence.push(gpii.dataLoader.createBatchUploadStep);
+    }
+    fluid.promise.sequence(sequence, options).then(
+        function (/*result*/) {
+            fluid.log("Done.");
+            process.exit(0);
+        },
+        function (error) {
+            fluid.log(error);
+            process.exit(1);
+        }
+    );
 };
-
 gpii.dataLoader.orchestrate();

From 9a090a1d89eac5da25733d8109b3ee1fe6f2b395 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 11 Sep 2018 14:43:32 -0400
Subject: [PATCH 31/41] GPII-3138:  Update snapsets in the database

Added code to update the views document in the database.
---
 scripts/deleteAndLoadSnapsets.js | 133 +++++++++++++++++++++++++++++--
 1 file changed, 125 insertions(+), 8 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 058d82e69..82ade382f 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -30,8 +30,8 @@ fluid.registerNamespace("gpii.dataLoader");
 fluid.setLogging(fluid.logLevel.INFO);
 
 // Handle command line
-if (process.argv.length < 5) {
-    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
+if (process.argv.length < 6) {
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
     process.exit(1);
 }
 
@@ -45,18 +45,23 @@ if (process.argv.length < 5) {
 gpii.dataLoader.initOptions = function (processArgv) {
     var dbOptions = {};
     dbOptions.couchDbUrl = processArgv[2];
-    dbOptions.buildDataDir = processArgv[3];
-    dbOptions.demoUserDir = processArgv[4];
-    if (processArgv.length > 5 && processArgv[5] === "--justDelete") { // for debugging.
+    dbOptions.staticDataDir = processArgv[3];
+    dbOptions.buildDataDir = processArgv[4];
+    dbOptions.demoUserDir = processArgv[5];
+    if (processArgv.length > 6 && processArgv[6] === "--justDelete") { // for debugging.
         dbOptions.justDelete = true;
     } else {
         dbOptions.justDelete = false;
     }
 
     // Set up database specific options
+    dbOptions.viewsUrl = dbOptions.couchDbUrl + "/_design/views";
     dbOptions.prefsSafesViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
     dbOptions.gpiiKeysViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
     dbOptions.parsedCouchDbUrl = url.parse(dbOptions.couchDbUrl);
+    dbOptions.staticData = [];
+    /* dbOptions.newViews; */
+    /* dbOptions.oldViews; */
     dbOptions.snapsetPrefsSafes = [];
     dbOptions.gpiiKeys = [];
     dbOptions.postOptions = {
@@ -76,10 +81,118 @@ gpii.dataLoader.initOptions = function (processArgv) {
         dbOptions.parsedCouchDbUrl.port +
         dbOptions.parsedCouchDbUrl.pathname + "'"
     );
+    fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
+    fluid.log("BUILD_DEMOUSER_DIR: '" + dbOptions.demoUserDir + "'");
     return dbOptions;
 };
 
+/*
+ * Loads the static data from disk, and creates a separate reference to the
+ * views document.
+ * @param {Object} options - Object that has the path to the directory
+ *                           containing the static data. On output, contains
+ *                           members for the new views and other static data.
+ */
+gpii.dataLoader.loadStaticDataFromDisk = function (options) {
+    var data = gpii.dataLoader.getDataFromDirectory(options.staticDataDir);
+    var views = fluid.find(data, function (anElement) {
+        if (anElement._id && anElement._id === "_design/views") {
+            return anElement;
+        } else {
+            return undefined;
+        }
+    });
+    options.staticData = data;
+    options.newViews = views;
+};
+
+/*
+ * Create the step that loads the static data into the database.
+ * @param {Object} options - Object that has the static data to load.
+ * @return {Promise} - A promise that resolves loading the static data.
+ */
+gpii.dataLoader.createStaticDataStep = function (options) {
+    var togo = fluid.promise();
+    var response = gpii.dataLoader.createResponseHandler(
+        function (responseString, options) {
+            fluid.log("Loading static data from '" + options.staticDataDir + "'");
+            return "Uploaded static data.";
+        },
+        options,
+        togo,
+        "Error loading static data into database: "
+    );
+    var staticDataRequest = gpii.dataLoader.createPostRequest(
+        options.staticData, response, options
+    );
+    staticDataRequest.end();
+    return togo;
+};
+
+/*
+ * Create the step that retrieves the current views from the database.
+ * @param {Object} options - Object containing the views URL into the database.
+ * @return {Promise} - A promise that resolves retrieving the old views.
+ */
+gpii.dataLoader.createFetchOldViewsStep = function (options) {
+    var togo = fluid.promise();
+    var response = gpii.dataLoader.createResponseHandler(
+        function (responseString, options) {
+            fluid.log("Retrieving old views from database.");
+            var oldViews = JSON.parse(responseString);
+            options.oldViews = oldViews;
+            return oldViews;
+        },
+        options,
+        togo,
+        "Error retrieving old views from database: "
+    );
+    var viewsRequest = gpii.dataLoader.queryDatabase(
+        options.viewsUrl, response, options
+    );
+    viewsRequest.end();
+    return togo;
+};
+
+/*
+ * Create the step that updates the views in the database.
+ * @param {Object} options - New views data to update with, and the old views
+ *                           currently in the database.
+ * @return {Promise} - A promise that resolves updating the views.
+ */
+gpii.dataLoader.createUpdateViewsStep = function (options) {
+    var togo = fluid.promise();
+
+    // Check to see if the views need updating.
+    // JS: Not sure how useful this is.
+    var oldViews = JSON.stringify(options.oldViews.views);
+    var newViews = JSON.stringify(options.newViews.views);
+    if (newViews === oldViews) {
+        fluid.log ("New views match old views, no change.");
+        togo.resolve("Updated views:  no change");
+    }
+    else {
+        var response = gpii.dataLoader.createResponseHandler(
+            function (responseString) {
+                var result = JSON.parse(responseString)[0];
+                fluid.log("Updated views: '" + JSON.stringify(result) + "'");
+                return JSON.stringify(result);
+            },
+            options,
+            togo,
+            "Error updating views: "
+        );
+        var viewsDataToPost = options.oldViews;         // id and rev
+        viewsDataToPost.views = options.newViews.views; // new data.
+        var request = gpii.dataLoader.createPostRequest(
+            [viewsDataToPost], response, options
+        );
+        request.end();
+    }
+    return togo;
+};
+
 /**
  * Find the Prefs Safes of type "snapset", mark them to be deleted and add
  * them to an array of records to remove.
@@ -155,7 +268,7 @@ gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, sn
  */
 gpii.dataLoader.configureBatchDelete = function (batchDeleteResponse, options) {
     var docsToRemove = options.snapsetPrefsSafes.concat(options.gpiiKeys);
-    return gpii.dataLoader.createBulkDocsRequest(
+    return gpii.dataLoader.createPostRequest(
         docsToRemove, batchDeleteResponse, options
     );
 };
@@ -167,7 +280,7 @@ gpii.dataLoader.configureBatchDelete = function (batchDeleteResponse, options) {
  * @param {Object} options - Data loader options, specifically the POST options.
  * @return {http.ClientRequest} - An http request object.
  */
-gpii.dataLoader.createBulkDocsRequest = function (dataToPost, responseHandler, options) {
+gpii.dataLoader.createPostRequest = function (dataToPost, responseHandler, options) {
     var batchPostData = JSON.stringify({"docs": dataToPost});
     options.postOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
     var batchDocsRequest = http.request(options.postOptions, responseHandler);
@@ -375,7 +488,7 @@ gpii.dataLoader.createBatchUploadStep = function (options) {
         options,
         togo
     );
-    var bulkUploadRequest = gpii.dataLoader.createBulkDocsRequest(allData, response, options);
+    var bulkUploadRequest = gpii.dataLoader.createPostRequest(allData, response, options);
     bulkUploadRequest.end();
     return togo;
 };
@@ -385,7 +498,11 @@ gpii.dataLoader.createBatchUploadStep = function (options) {
  */
 gpii.dataLoader.orchestrate = function () {
     var options = gpii.dataLoader.initOptions(process.argv);
+    gpii.dataLoader.loadStaticDataFromDisk(options);
     var sequence = [
+        gpii.dataLoader.createStaticDataStep,
+        gpii.dataLoader.createFetchOldViewsStep,
+        gpii.dataLoader.createUpdateViewsStep,
         gpii.dataLoader.createFetchSnapsetsStep,
         gpii.dataLoader.createFetchGpiiKeysStep,
         gpii.dataLoader.createBatchDeleteStep

From a4cf4525d30d27fb1fadad4fccb9a5fe42f483fe Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 12 Sep 2018 16:26:41 -0400
Subject: [PATCH 32/41] GPII-3138:  Update snapsets in the database

Removed all the demo user preferences used in the Aug demo:  the
20 empty preference sets and the "roy" and "quint" preferences.
Also, modified the data loader to not load these preferences into
the database.
---
 package.json                                  |  2 +-
 scripts/deleteAndLoadSnapsets.js              | 35 ++++++------
 ...129bc341-0025-417e-8107-cb567ab419d0.json5 | 53 -------------------
 ...132922e6-569b-404c-9db2-c207544cc959.json5 | 52 ------------------
 ...20558561-dc5b-4973-85f3-9bca975090cb.json5 | 11 ----
 ...226722c6-8895-40b4-9297-424b9da9c340.json5 | 11 ----
 ...2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5 | 11 ----
 ...2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5 | 11 ----
 ...2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5 | 11 ----
 ...339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5 | 11 ----
 ...587ad68c-6470-486b-9828-6ee224ae6902.json5 | 11 ----
 ...7009c709-9ce0-441f-9b56-aec7827a1d5c.json5 | 11 ----
 ...80489c34-50cd-43db-a1bc-9a50960d5b3a.json5 | 11 ----
 ...8060c500-6be9-462a-8ae1-8705be497916.json5 | 11 ----
 ...8070295c-d4dc-4b02-9113-ac08f5b0484a.json5 | 11 ----
 ...881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5 | 11 ----
 ...a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5 | 11 ----
 ...b7a438ab-cf2a-472d-9a96-5d87ee187770.json5 | 11 ----
 ...c3657704-04f6-4f02-a777-c361f0110876.json5 | 11 ----
 ...cd5ca821-6a6e-4734-b956-83546bae1a5f.json5 | 11 ----
 ...cead392d-05da-4ecc-9bd3-88c789f94d8c.json5 | 11 ----
 ...dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5 | 11 ----
 ...e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5 | 11 ----
 ...e8c62569-ac93-4c45-8085-f172ca570311.json5 | 11 ----
 24 files changed, 16 insertions(+), 346 deletions(-)
 delete mode 100644 testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
 delete mode 100644 testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
 delete mode 100644 testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
 delete mode 100644 testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
 delete mode 100644 testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
 delete mode 100644 testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
 delete mode 100644 testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
 delete mode 100644 testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
 delete mode 100644 testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
 delete mode 100644 testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
 delete mode 100644 testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
 delete mode 100644 testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
 delete mode 100644 testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
 delete mode 100644 testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
 delete mode 100644 testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
 delete mode 100644 testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
 delete mode 100644 testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
 delete mode 100644 testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
 delete mode 100644 testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
 delete mode 100644 testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
 delete mode 100644 testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
 delete mode 100644 testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5

diff --git a/package.json b/package.json
index 9e33a00ad..ce1195abe 100644
--- a/package.json
+++ b/package.json
@@ -78,6 +78,6 @@
         "test:vagrantProduction": "vagrant ssh -c 'cd /home/vagrant/sync/universal; DISPLAY=:0; ./scripts/vagrantCloudBasedContainers.sh'",
         "posttest": "node node_modules/nyc/bin/nyc.js report -r text-summary -r html --report-dir reports --temp-directory coverage",
         "start": "node gpii.js",
-        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset && node scripts/convertPrefs.js testData/preferences/ build/dbData/user/ user && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/ user && node scripts/convertPrefs.js testData/preferences/demoUserPrefs/ build/dbData/demouser/ user"
+        "postinstall": "node scripts/browserifyTestDependency.js && node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset && node scripts/convertPrefs.js testData/preferences/ build/dbData/user/ user && node scripts/convertPrefs.js tests/data/preferences/ build/tests/dbData/ user"
     }
 }
diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 82ade382f..341d83838 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -9,13 +9,13 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 */
 
 // This script modifies the preferences database:
-// 1. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
-// 2. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
-// 3. Deletes these Prefs Safes and their associated GPII Keys from the database,
-// 4. Uploads the new Prefs Safes and their GPII Keys to the database,
-// 5. Uploads demo user Prefs Safes and their Keys in to the database, if they are not already present.
+// 1. Update the views records for accessing Prefs Safes and GPII Keys,
+// 2. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
+// 3. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
+// 4. Deletes these Prefs Safes and their associated GPII Keys from the database,
+// 5. Uploads the new Prefs Safes and their GPII Keys to the database,
 // A sample command that runs this script:
-// node deleteAndLoadSnapsets.js $COUCHDBURL $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR
+// node deleteAndLoadSnapsets.js $COUCHDBURL $STATIC_DATA_DIR $BUILD_DATA_DIR
 
 "use strict";
 
@@ -30,8 +30,8 @@ fluid.registerNamespace("gpii.dataLoader");
 fluid.setLogging(fluid.logLevel.INFO);
 
 // Handle command line
-if (process.argv.length < 6) {
-    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR [--justDelete]");
+if (process.argv.length < 5) {
+    fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR [--justDelete]");
     process.exit(1);
 }
 
@@ -47,8 +47,7 @@ gpii.dataLoader.initOptions = function (processArgv) {
     dbOptions.couchDbUrl = processArgv[2];
     dbOptions.staticDataDir = processArgv[3];
     dbOptions.buildDataDir = processArgv[4];
-    dbOptions.demoUserDir = processArgv[5];
-    if (processArgv.length > 6 && processArgv[6] === "--justDelete") { // for debugging.
+    if (processArgv.length > 5 && processArgv[5] === "--justDelete") { // for debugging.
         dbOptions.justDelete = true;
     } else {
         dbOptions.justDelete = false;
@@ -83,7 +82,6 @@ gpii.dataLoader.initOptions = function (processArgv) {
     );
     fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
-    fluid.log("BUILD_DEMOUSER_DIR: '" + dbOptions.demoUserDir + "'");
     return dbOptions;
 };
 
@@ -467,28 +465,25 @@ gpii.dataLoader.createBatchDeleteStep = function (options) {
  */
 gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
     fluid.log("Bulk loading of build data from '" + options.buildDataDir + "'");
-    fluid.log("Bulk loading of demo user data from '" + options.demoUserDir + "'");
-    return "Uploaded latest snapsets and demo user preferences";
+    return "Uploaded latest snapsets preferences";
 };
 
 /*
- * Create the step that uploads, in batch, the new snapset Prefs Safes, their
- * associated GPII keys, and the demo user Prefs Safes/GPII Keys.
- * @param {Object} options - Object that has the paths to the directories that
- *                           contain the new snapsets and demo user preferences.
+ * Create the step that uploads, in batch, the new snapset Prefs Safes, and
+ * their associated GPII keys.
+ * @param {Object} options - Object that has the paths to the directory that
+ *                           contains the latest snapsets.
  * @return {Promise} - A promise that resolves the upload.
  */
 gpii.dataLoader.createBatchUploadStep = function (options) {
     var togo = fluid.promise();
     var buildData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
-    var demoUserData = gpii.dataLoader.getDataFromDirectory(options.demoUserDir);
-    var allData = buildData.concat(demoUserData);
     var response = gpii.dataLoader.createResponseHandler(
         gpii.dataLoader.logSnapsetsUpload,
         options,
         togo
     );
-    var bulkUploadRequest = gpii.dataLoader.createPostRequest(allData, response, options);
+    var bulkUploadRequest = gpii.dataLoader.createPostRequest(buildData, response, options);
     bulkUploadRequest.end();
     return togo;
 };
diff --git a/testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5 b/testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
deleted file mode 100644
index 539d0cbff..000000000
--- a/testData/preferences/demoUserPrefs/129bc341-0025-417e-8107-cb567ab419d0.json5
+++ /dev/null
@@ -1,53 +0,0 @@
-// Quint: switches the OS to Spanish, turns on JAWS with a Spanish voice.
-{
-    "flat": {
-        "name": "Quint",
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {
-                    "http://registry.gpii.net/common/language": "es-ES",
-                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
-                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
-                        "Options.PrimarySynthesizer" : "eloq",
-                        "options.SayAllIndicateCaps" : false,
-                        "ENU-Global.Rate" : 50,
-                        "ENU-Global.Pitch" : 75,
-                        "ENU-Global.Volume" : 100,
-                        "ENU-Global.Punctuation" : 2,
-                        "ENU-JAWSCursor.Rate": 50,
-                        "ENU-Keyboard.Rate": 50,
-                        "ENU-MenuAndDialog.Rate": 50,
-                        "ENU-Message.Rate": 50,
-                        "ENU-PCCursor.Rate": 50,
-                        "ENU-JAWSCursor.Pitch": 75,
-                        "ENU-Keyboard.Pitch": 75,
-                        "ENU-MenuAndDialog.Pitch": 75,
-                        "ENU-Message.Pitch": 75,
-                        "ENU-PCCursor.Pitch": 75,
-                        "ENU-JAWSCursor.Volume": 100,
-                        "ENU-Keyboard.Volume": 100,
-                        "ENU-MenuAndDialog.Volume": 100,
-                        "ENU-Message.Volume": 100,
-                        "ENU-PCCursor.Volume": 100,
-                        "ENU-JAWSCursor.Punctuation": 2,
-                        "ENU-Keyboard.Punctuation": 2,
-                        "ENU-MenuAndDialog.Punctuation": 2,
-                        "ENU-Message.Punctuation": 2,
-                        "ENU-PCCursor.Punctuation": 2,
-                        "ENU-Global.SynthLangString": "Castilian Spanish",
-                        "ENU-JAWSCursor.SynthLangString": "Castilian Spanish",
-                        "ENU-Keyboard.SynthLangString": "Castilian Spanish",
-                        "ENU-MenuAndDialog.SynthLangString": "Castilian Spanish",
-                        "ENU-Message.SynthLangString": "Castilian Spanish",
-                        "ENU-PCCursor.SynthLangString": "Castilian Spanish",
-                        "options.TypingEcho": 3,
-                        "options.SayAllMode": 0,
-                        "Braille.BrailleMode": 0,
-                        "options.SayAllIgnoreShiftKeys": true
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5 b/testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
deleted file mode 100644
index b4642bb85..000000000
--- a/testData/preferences/demoUserPrefs/132922e6-569b-404c-9db2-c207544cc959.json5
+++ /dev/null
@@ -1,52 +0,0 @@
-// Roy: turns on JAWS and speaks relatively fast.
-{
-    "flat": {
-        "name": "Roy",
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {
-                    "http://registry.gpii.net/applications/com.freedomscientific.jaws": {
-                        "Voice Profiles.ActiveVoiceProfileName" : "GPII",
-                        "Options.PrimarySynthesizer" : "eloq",
-                        "options.SayAllIndicateCaps" : false,
-                        "ENU-Global.Rate" : 100,
-                        "ENU-Global.Pitch" : 65,
-                        "ENU-Global.Volume" : 100,
-                        "ENU-Global.Punctuation" : 2,
-                        "ENU-JAWSCursor.Rate": 100,
-                        "ENU-Keyboard.Rate": 100,
-                        "ENU-MenuAndDialog.Rate": 100,
-                        "ENU-Message.Rate": 100,
-                        "ENU-PCCursor.Rate": 100,
-                        "ENU-JAWSCursor.Pitch": 65,
-                        "ENU-Keyboard.Pitch": 65,
-                        "ENU-MenuAndDialog.Pitch": 65,
-                        "ENU-Message.Pitch": 65,
-                        "ENU-PCCursor.Pitch": 65,
-                        "ENU-JAWSCursor.Volume": 100,
-                        "ENU-Keyboard.Volume": 100,
-                        "ENU-MenuAndDialog.Volume": 100,
-                        "ENU-Message.Volume": 100,
-                        "ENU-PCCursor.Volume": 100,
-                        "ENU-JAWSCursor.Punctuation": 2,
-                        "ENU-Keyboard.Punctuation": 2,
-                        "ENU-MenuAndDialog.Punctuation": 2,
-                        "ENU-Message.Punctuation": 2,
-                        "ENU-PCCursor.Punctuation": 2,
-                        "ENU-Global.SynthLangString": "American English",
-                        "ENU-JAWSCursor.SynthLangString": "American English",
-                        "ENU-Keyboard.SynthLangString": "American English",
-                        "ENU-MenuAndDialog.SynthLangString": "American English",
-                        "ENU-Message.SynthLangString": "American English",
-                        "ENU-PCCursor.SynthLangString": "American English",
-                        "options.TypingEcho": 3,
-                        "options.SayAllMode": 0,
-                        "Braille.BrailleMode": 0,
-                        "options.SayAllIgnoreShiftKeys": true
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5 b/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/20558561-dc5b-4973-85f3-9bca975090cb.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5 b/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/226722c6-8895-40b4-9297-424b9da9c340.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5 b/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/2361771e-6d77-4fc3-9fa6-4b5e1ef04c4c.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5 b/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/2362cbe8-6b59-4682-a7a3-8e9f07a178a3.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5 b/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/2ce4ba4b-704d-4c5f-9dae-d42ae7de535e.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5 b/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/339926ee-d7e3-48f8-8bc4-4c9773cc1656.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5 b/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/587ad68c-6470-486b-9828-6ee224ae6902.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5 b/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/7009c709-9ce0-441f-9b56-aec7827a1d5c.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5 b/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/80489c34-50cd-43db-a1bc-9a50960d5b3a.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5 b/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/8060c500-6be9-462a-8ae1-8705be497916.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5 b/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/8070295c-d4dc-4b02-9113-ac08f5b0484a.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5 b/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/881bb4c5-1b9e-43f2-a4d4-994137fc0212.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5 b/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/a33a7cdf-1085-4838-95c5-8a4eb9008dd1.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5 b/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/b7a438ab-cf2a-472d-9a96-5d87ee187770.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5 b/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/c3657704-04f6-4f02-a777-c361f0110876.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5 b/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/cd5ca821-6a6e-4734-b956-83546bae1a5f.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5 b/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/cead392d-05da-4ecc-9bd3-88c789f94d8c.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5 b/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/dfd1b660-fdff-47ef-aea2-71cdd23283d3.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5 b/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/e48c8cea-9719-45a3-88d5-5d33e1a7e357.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}
diff --git a/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5 b/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
deleted file mode 100644
index 98cd9afad..000000000
--- a/testData/preferences/demoUserPrefs/e8c62569-ac93-4c45-8085-f172ca570311.json5
+++ /dev/null
@@ -1,11 +0,0 @@
-// Empty user key
-{
-    "flat": {
-        "contexts": {
-            "gpii-default": {
-                "name": "Default preferences",
-                "preferences": {}
-            }
-        }
-    }
-}

From 63c1a115715835ac05f536723242f2b3bd36dba3 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Thu, 13 Sep 2018 11:30:07 -0400
Subject: [PATCH 33/41] GPII-3138:  Update snapsets in the database

Removed another reference to the demo user preferences directory
that should have been removed in last commit.
---
 scripts/vagrantCloudBasedContainers.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/vagrantCloudBasedContainers.sh b/scripts/vagrantCloudBasedContainers.sh
index 340ca8981..49bcc8d4f 100755
--- a/scripts/vagrantCloudBasedContainers.sh
+++ b/scripts/vagrantCloudBasedContainers.sh
@@ -41,7 +41,6 @@ fi
 UNIVERSAL_DIR="/home/vagrant/sync/universal"
 STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
 BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData/snapset"
-BUILD_DEMOUSER_DIR="$UNIVERSAL_DIR/build/dbData/demouser"
 
 DATALOADER_IMAGE="herrclown/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
@@ -83,7 +82,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -v $BUILD_DEMOUSER_DIR:/build_data/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

From 7ce1b84d3afdfb62280e53b8a3ec1692f9811988 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 18 Sep 2018 15:02:11 -0400
Subject: [PATCH 34/41] GPII-3138:  Update snapsets in the database

Added authorization for cases where a userid/password is needed
for accessing the database.
---
 scripts/deleteAndLoadSnapsets.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 341d83838..baa83d962 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -74,6 +74,11 @@ gpii.dataLoader.initOptions = function (processArgv) {
             "Content-Type": "application/json"
         }
     };
+    if (dbOptions.parsedCouchDbUrl.username) {
+        dbOptions.postOptions.auth =
+            dbOptions.parsedCouchDbUrl.username + ":" +
+            dbOptions.parsedCouchDbUrl.password;
+    }
     fluid.log("COUCHDB_URL: '" +
         dbOptions.parsedCouchDbUrl.protocol + "//" +
         dbOptions.parsedCouchDbUrl.hostname + ":" +

From afc68885211e330565faf5a736645e871eef36de Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 19 Sep 2018 11:01:51 -0400
Subject: [PATCH 35/41] GPII-3138:  Update snapsets in the database

Temporary addition of log messages showing the username and password
used to access the database, but, also, logging whem the static data
has been retrieved from disk.
---
 scripts/deleteAndLoadSnapsets.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index baa83d962..2504a7219 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -87,6 +87,14 @@ gpii.dataLoader.initOptions = function (processArgv) {
     );
     fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
+
+    //***** For Debugging AWS.  To Be Removed.
+    fluid.log("DEBUG - CouchDB URL with secrets: '" + dbOptions.couchDbUrl + "'");
+    fluid.log("DEBUG - Parsed CouchDB username '" + dbOptions.parsedCouchDbUrl.username + "'");
+    fluid.log("DEBUG - Parsed CouchDB password '" + dbOptions.parsedCouchDbUrl.password + "'");
+    fluid.log("DEBUG - POST options auth: '" + dbOptions.postOptions.auth + "'");
+    //*****
+
     return dbOptions;
 };
 
@@ -108,6 +116,8 @@ gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     });
     options.staticData = data;
     options.newViews = views;
+    fluid.log("Loaded static data from: '" + options.staticDataDir + "'");
+    fluid.log("\views data '" + ( views ? "loaded." : "missing." ) + "'");
 };
 
 /*

From 75456babcfb1e1883bcdacc1d7d07b242d4062a5 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 19 Sep 2018 11:25:29 -0400
Subject: [PATCH 36/41] GPII-3138:  Update snapsets in the database

Modified such that POST "auth" field uses "auth" field of parsed URL.
---
 scripts/deleteAndLoadSnapsets.js | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 2504a7219..2cb6ac945 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -74,10 +74,8 @@ gpii.dataLoader.initOptions = function (processArgv) {
             "Content-Type": "application/json"
         }
     };
-    if (dbOptions.parsedCouchDbUrl.username) {
-        dbOptions.postOptions.auth =
-            dbOptions.parsedCouchDbUrl.username + ":" +
-            dbOptions.parsedCouchDbUrl.password;
+    if (dbOptions.parsedCouchDbUrl.auth) {
+        dbOptions.postOptions.auth = dbOptions.parsedCouchDbUrl.auth;
     }
     fluid.log("COUCHDB_URL: '" +
         dbOptions.parsedCouchDbUrl.protocol + "//" +
@@ -116,8 +114,8 @@ gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     });
     options.staticData = data;
     options.newViews = views;
-    fluid.log("Loaded static data from: '" + options.staticDataDir + "'");
-    fluid.log("\views data '" + ( views ? "loaded." : "missing." ) + "'");
+    fluid.log("Retrieved static data from: '" + options.staticDataDir + "'");
+    fluid.log("\tviews data '" + ( views ? "retrieved." : "missing." ) + "'");
 };
 
 /*

From 230c8f078f508ad3189a460d9567840db151150c Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 19 Sep 2018 13:50:57 -0400
Subject: [PATCH 37/41] GPII-3138:  Update snapsets in the database

Removed messages for debugging username/password access to
the databse.
---
 scripts/deleteAndLoadSnapsets.js | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 2cb6ac945..1495cbe9d 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -67,6 +67,7 @@ gpii.dataLoader.initOptions = function (processArgv) {
         hostname: dbOptions.parsedCouchDbUrl.hostname,
         port: dbOptions.parsedCouchDbUrl.port,
         path: "/gpii/_bulk_docs",
+        auth: dbOptions.parsedCouchDbUrl.auth,
         method: "POST",
         headers: {
             "Accept": "application/json",
@@ -74,9 +75,6 @@ gpii.dataLoader.initOptions = function (processArgv) {
             "Content-Type": "application/json"
         }
     };
-    if (dbOptions.parsedCouchDbUrl.auth) {
-        dbOptions.postOptions.auth = dbOptions.parsedCouchDbUrl.auth;
-    }
     fluid.log("COUCHDB_URL: '" +
         dbOptions.parsedCouchDbUrl.protocol + "//" +
         dbOptions.parsedCouchDbUrl.hostname + ":" +
@@ -86,13 +84,6 @@ gpii.dataLoader.initOptions = function (processArgv) {
     fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
     fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
 
-    //***** For Debugging AWS.  To Be Removed.
-    fluid.log("DEBUG - CouchDB URL with secrets: '" + dbOptions.couchDbUrl + "'");
-    fluid.log("DEBUG - Parsed CouchDB username '" + dbOptions.parsedCouchDbUrl.username + "'");
-    fluid.log("DEBUG - Parsed CouchDB password '" + dbOptions.parsedCouchDbUrl.password + "'");
-    fluid.log("DEBUG - POST options auth: '" + dbOptions.postOptions.auth + "'");
-    //*****
-
     return dbOptions;
 };
 
@@ -115,7 +106,7 @@ gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     options.staticData = data;
     options.newViews = views;
     fluid.log("Retrieved static data from: '" + options.staticDataDir + "'");
-    fluid.log("\tviews data '" + ( views ? "retrieved." : "missing." ) + "'");
+    fluid.log("\tViews data " + ( views ? "retrieved." : "missing." ));
 };
 
 /*

From 33f1741b0038570862cbdeeb77a503a8b78415d7 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Tue, 9 Oct 2018 14:30:51 -0400
Subject: [PATCH 38/41] GPII-3138:  Update snapsets in the database.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Modified based on Cindy's review comments:
- updated README.md and README.txt to better explain the different
  kinds of prefences data and their usage
- modified covertprefs.js documentation, usage, and to only
  ´accept/create "snapset" or "user" preferences
- refactored deleteAndLoadSnapsets.js to place some of the common
  functionality in a new function.
---
 README.md                        |  22 ++--
 scripts/convertPrefs.js          |   9 +-
 scripts/deleteAndLoadSnapsets.js | 176 ++++++++++++++++---------------
 testData/dbData/README.txt       |  22 ++--
 4 files changed, 125 insertions(+), 104 deletions(-)

diff --git a/README.md b/README.md
index 685edea18..5af77b6cb 100644
--- a/README.md
+++ b/README.md
@@ -98,12 +98,22 @@ have set the `NODE_ENV` variable.
 
 ### Convert Preferences Data
 
-GPII has 2 set of preferences JSON5 data files:
-
-* The preferences files for running GPII are located at %gpii-universal/testData/preferences
-* The preferences files for running node tests are located at %gpii-universal/tests/data/preferences
-
-When any preferences file in either one of these 2 directories are modified, running `npm run postinstall` will generate
+GPII has two sets of preferences JSON5 data files, located at `%gpii-universal/testData/preferences` and `%gpii-universal/tests/data/preferences`:
+
+* The preferences files for running GPII and for integration tests are located at
+  `%gpii-universal/testData/preferences`.  These files are converted into two types of preferences safes and GPII keys:
+  * They are converted into `snapset` preferences safes and GPII keys and placed into the
+    `%gpii-universal/build/dbData/snapset/` folder.   These are used to update the snapsets in CouchDB when GPII is
+    run in a production or staging configuration.
+  * They are also converted into `user` preferences safes and GPII keys and placed into the
+    `%gpii-universal/build/dbData/user/` folder.   These are used with PouchDB when GPII runs in a development
+    configuration or when running GPII integration tests.
+* The preferences files for running node tests are located at `%gpii-universal/tests/data/preferences`.  These files
+  are converted into preferences safes and GPII keys:
+  * They are converted into `user` preferences safes and GPII keys and placed into the
+    `%gpii-universal/build/tests/dbData/user` folder.   These are used during testing with PouchDB.
+
+When any preferences file in either one of these two directories are modified, running `npm run postinstall` will generate
 gpiiKeys.json and prefsSafes.json, the files that are in the structure to be loaded into PouchDB/CouchDB, based off
 these directories. This step is needed for the modification to be applied to GPII.
 
diff --git a/scripts/convertPrefs.js b/scripts/convertPrefs.js
index 4a6892e2f..d3429cabc 100644
--- a/scripts/convertPrefs.js
+++ b/scripts/convertPrefs.js
@@ -11,7 +11,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 // This script reads files from an input directory that contains preferences JSON5 files and convert them to JSON files of GPII keys and
 // preferences safes suitable for direct loading into CouchDB or PouchDB, which comply with the new GPII data model:
 // https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences in the target directory
-// Usage: node scripts/convertPrefs.js {input_path} {target_path}
+// Usage: node scripts/convertPrefs.js {input_path} {target_path} {prefsSafeType}, where {prefsSafeType} is one of "snapset" or "user" and defaults to "user"
 //
 // A sample command that runs this script in the universal root directory:
 // node scripts/convertPrefs.js testData/preferences/ build/dbData/ snapset
@@ -25,8 +25,13 @@ var fs = require("fs"),
 
 var inputDir = process.argv[2];
 var targetDir = process.argv[3];
-var prefsSafeType = process.argv[4];
+var prefsSafeType = process.argv[4] || "user";
 
+if (prefsSafeType !== "snapset" && prefsSafeType !== "user") {
+    console.log("Usage: node scripts/convertPrefs.js InputFolder OutputFolder PrefsSafeType");
+    console.log("  where PrefsSafeType, is one of 'snapset' or 'user' (defaults to 'user')");
+    process.exit(1);
+}
 var prefsSafes = [];
 var gpiiKeys = [];
 
diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 1495cbe9d..10ce6785e 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -16,6 +16,11 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 // 5. Uploads the new Prefs Safes and their GPII Keys to the database,
 // A sample command that runs this script:
 // node deleteAndLoadSnapsets.js $COUCHDBURL $STATIC_DATA_DIR $BUILD_DATA_DIR
+//
+// There is also an optional final [--justDelete] argument for testing/debugging.
+// If present, the script exits with a zero exit status after deleting all the
+// snapset PrefsSafes and their GPII keys.  That is, the script does only the
+// first four steps listed above.
 
 "use strict";
 
@@ -109,27 +114,53 @@ gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     fluid.log("\tViews data " + ( views ? "retrieved." : "missing." ));
 };
 
+/*
+ * Utility to configure a step:  creates a response callback, binds it to an
+ * http database request, and configures a promise to resolve/reject when the
+ * response callback finishes or fails.
+ * @param {Object} details - Specific information for the request and response,
+ *                           such as error messages, urls, POST data, and so on.
+ *                           These details are set appropriately by the caller.
+ * @param {Object} options - Additional options.
+ * @return {Promise} - A promise that resolves the configured step.
+ */
+gpii.dataLoader.configureStep = function (details, options) {
+    var togo = fluid.promise();
+    var response = gpii.dataLoader.createResponseHandler(
+        details.responseDataHandler,
+        options,
+        togo,
+        details.responseErrMsg
+    );
+    var request;
+    if (details.dataToPost) {
+        request = gpii.dataLoader.createPostRequest(
+            details.dataToPost, response, options
+        );
+    } else {
+        request = gpii.dataLoader.queryDatabase(
+            details.requestUrl, response, details.requestErrMsg
+        );
+    }
+    request.end();
+    return togo;
+};
+
 /*
  * Create the step that loads the static data into the database.
  * @param {Object} options - Object that has the static data to load.
  * @return {Promise} - A promise that resolves loading the static data.
  */
 gpii.dataLoader.createStaticDataStep = function (options) {
-    var togo = fluid.promise();
-    var response = gpii.dataLoader.createResponseHandler(
-        function (responseString, options) {
+    var details = {
+        dataToPost: options.staticData,
+        responseDataHandler: function (responseString, options) {
             fluid.log("Loading static data from '" + options.staticDataDir + "'");
             return "Uploaded static data.";
         },
-        options,
-        togo,
-        "Error loading static data into database: "
-    );
-    var staticDataRequest = gpii.dataLoader.createPostRequest(
-        options.staticData, response, options
-    );
-    staticDataRequest.end();
-    return togo;
+        responseErrMsg: "Error loading static data into database: "
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
@@ -138,23 +169,18 @@ gpii.dataLoader.createStaticDataStep = function (options) {
  * @return {Promise} - A promise that resolves retrieving the old views.
  */
 gpii.dataLoader.createFetchOldViewsStep = function (options) {
-    var togo = fluid.promise();
-    var response = gpii.dataLoader.createResponseHandler(
-        function (responseString, options) {
+    var details = {
+        requestUrl: options.viewsUrl,
+        requestErrMsg: "Error requesting old views from database: ",
+        responseDataHandler: function (responseString, options) {
             fluid.log("Retrieving old views from database.");
             var oldViews = JSON.parse(responseString);
             options.oldViews = oldViews;
             return oldViews;
         },
-        options,
-        togo,
-        "Error retrieving old views from database: "
-    );
-    var viewsRequest = gpii.dataLoader.queryDatabase(
-        options.viewsUrl, response, options
-    );
-    viewsRequest.end();
-    return togo;
+        responseErrMsg: "Error retrieving old views from database: "
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
@@ -164,35 +190,30 @@ gpii.dataLoader.createFetchOldViewsStep = function (options) {
  * @return {Promise} - A promise that resolves updating the views.
  */
 gpii.dataLoader.createUpdateViewsStep = function (options) {
-    var togo = fluid.promise();
-
     // Check to see if the views need updating.
     // JS: Not sure how useful this is.
     var oldViews = JSON.stringify(options.oldViews.views);
     var newViews = JSON.stringify(options.newViews.views);
     if (newViews === oldViews) {
-        fluid.log ("New views match old views, no change.");
+        var togo = fluid.promise();
+        fluid.log("New views match old views, no change.");
         togo.resolve("Updated views:  no change");
+        return togo;
     }
     else {
-        var response = gpii.dataLoader.createResponseHandler(
-            function (responseString) {
+        var viewsDataToPost = options.oldViews;         // id and rev
+        viewsDataToPost.views = options.newViews.views; // new data.
+        var details = {
+            dataToPost: [viewsDataToPost],
+            responseDataHandler: function (responseString) {
                 var result = JSON.parse(responseString)[0];
                 fluid.log("Updated views: '" + JSON.stringify(result) + "'");
                 return JSON.stringify(result);
             },
-            options,
-            togo,
-            "Error updating views: "
-        );
-        var viewsDataToPost = options.oldViews;         // id and rev
-        viewsDataToPost.views = options.newViews.views; // new data.
-        var request = gpii.dataLoader.createPostRequest(
-            [viewsDataToPost], response, options
-        );
-        request.end();
+            responseErrMsg: "Error updating views: "
+        };
+        return gpii.dataLoader.configureStep(details, options);
     }
-    return togo;
 };
 
 /**
@@ -303,6 +324,9 @@ gpii.dataLoader.createPostRequest = function (dataToPost, responseHandler, optio
  * @return {Function} - Function reponse callback for an http request.
  */
 gpii.dataLoader.createResponseHandler = function (handleEnd, options, promise, errorMsg) {
+    if (!errorMsg) {
+        errorMsg = "";
+    }
     return function (response) {
         var responseString = "";
 
@@ -382,20 +406,13 @@ gpii.dataLoader.getDataFromDirectory = function (dataDir) {
  *                     currently in the database.
  */
 gpii.dataLoader.createFetchSnapsetsStep = function (options) {
-    var togo = fluid.promise();
-    var response = gpii.dataLoader.createResponseHandler(
-        gpii.dataLoader.processSnapsets,
-        options,
-        togo,
-        "Error retrieving snapsets Prefs Safes: "
-    );
-    var snapSetsRequest = gpii.dataLoader.queryDatabase(
-        options.prefsSafesViewUrl,
-        response,
-        "Error requesting snapsets Prefs Safes: "
-    );
-    snapSetsRequest.end();
-    return togo;
+    var details = {
+        requestUrl: options.prefsSafesViewUrl,
+        requestErrMsg: "Error requesting snapsets Prefs Safes: ",
+        responseDataHandler: gpii.dataLoader.processSnapsets,
+        responseErrMsg: "Error retrieving snapsets Prefs Safes: "
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
@@ -407,20 +424,13 @@ gpii.dataLoader.createFetchSnapsetsStep = function (options) {
  *                     database that correspond to snapset PrefsSafes.
  */
 gpii.dataLoader.createFetchGpiiKeysStep = function (options) {
-    var togo = fluid.promise();
-    var response = gpii.dataLoader.createResponseHandler(
-        gpii.dataLoader.processGpiiKeys,
-        options,
-        togo,
-        "Error finding snapset Prefs Safes associated GPII Keys: "
-    );
-    var request = gpii.dataLoader.queryDatabase(
-        options.gpiiKeysViewUrl,
-        response,
-        "Error requesting GPII Keys: "
-    );
-    request.end();
-    return togo;
+    var details = {
+        requestUrl: options.gpiiKeysViewUrl,
+        requestErrMsg: "Error requesting GPII Keys: ",
+        responseDataHandler: gpii.dataLoader.processGpiiKeys,
+        responseErrMsg: "Error finding snapset Prefs Safes associated GPII Keys: "
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
@@ -450,15 +460,11 @@ gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
  * @return {Promise} - The promise that resolves the deletion.
  */
 gpii.dataLoader.createBatchDeleteStep = function (options) {
-    var togo = fluid.promise();
-    var response = gpii.dataLoader.createResponseHandler(
-        gpii.dataLoader.logSnapsetDeletion,
-        options,
-        togo
-    );
-    var batchDeleteRequest = gpii.dataLoader.configureBatchDelete(response, options);
-    batchDeleteRequest.end();
-    return togo;
+    var details = {
+        dataToPost: options.snapsetPrefsSafes.concat(options.gpiiKeys),
+        responseDataHandler: gpii.dataLoader.logSnapsetDeletion
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
@@ -480,16 +486,12 @@ gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
  * @return {Promise} - A promise that resolves the upload.
  */
 gpii.dataLoader.createBatchUploadStep = function (options) {
-    var togo = fluid.promise();
-    var buildData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
-    var response = gpii.dataLoader.createResponseHandler(
-        gpii.dataLoader.logSnapsetsUpload,
-        options,
-        togo
-    );
-    var bulkUploadRequest = gpii.dataLoader.createPostRequest(buildData, response, options);
-    bulkUploadRequest.end();
-    return togo;
+    var newSnapsetsData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
+    var details = {
+        dataToPost: newSnapsetsData,
+        responseDataHandler: gpii.dataLoader.logSnapsetsUpload
+    };
+    return gpii.dataLoader.configureStep(details, options);
 };
 
 /*
diff --git a/testData/dbData/README.txt b/testData/dbData/README.txt
index 2c4c47200..1a9b26744 100644
--- a/testData/dbData/README.txt
+++ b/testData/dbData/README.txt
@@ -1,13 +1,17 @@
-This folder contains the DB data that is used for running GPII in a development config. They are loaded into:
-1. PouchDB when GPII runs in the development config;
-2. CouchDB at deploying the GPII cloud that is used for the testing purpose.
+This folder contains the DB data that is used for running GPII in different configurations. They are loaded into:
+1. PouchDB when GPII runs in a development configuration or when running GPII integration tests;
+2. CouchDB when GPII runs in a production or staging configuration.
 
-Each data file is corresponding to a document structure specified in the documentation of [GPII Data Model](https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences):
+Each data file corresponds to a document structure specified in the [GPII Data Model](https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences) documentation:
 
-* %universal/build/dbData/snapset/gpiiKeys.json: Contains all GPII keys for snapset preferences safes.
-* %universal/build/dbData/snapset/prefsSafes.json: Contains all snapset preferences safes corresponding to snapset GPII keys.
-* %universal/build/dbData/user/gpiiKeys.json: Contains all GPII keys for user preferences safes.
-* %universal/build/dbData/user/prefsSafes.json: Contains all user preferences safes corresponding to user GPII keys.
 * clientCredentials.json: Contains OAuth2 client credentials.
 * gpiiAppInstallationClients.json: Contains the information of all registered GPII app installations.
-* views.json: The PouchDB/CouchDB view functions.
+* views.json: The PouchDB/CouchDB "_design/views" functions.
+
+Additional preferences DB data is found elsewhere and used in conjunction with the files in this folder.  The preferences files located in %universal/testData/preferences/ are converted into both "snapset" and "user" preferences.  The following "snapset" folder contain snapset PrefsSafes and GPII keys for (1) running GPII in production and staging GPII configurations where the data is loaded into CouchDB, and (2) running GPII in its development configuration where the data is loaded into the local PouchDB:
+* %universal/build/dbData/snapset/gpiiKeys.json
+* %universal/build/dbData/snapset/prefsSafes.json
+
+The following "user" folder contains writable "user" data, loaded into the local PouchDB when running GPII integration tests:
+* %universal/build/dbData/user/gpiiKeys.json
+* %universal/build/dbData/user/prefsSafes.json

From 2c48ca5217442ea94263e1d9148a419db008b21b Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Thu, 11 Oct 2018 16:50:06 -0400
Subject: [PATCH 39/41] GPII-3138:  Update snapsets in the database

Modified based on Antranig's comments:
- modified the JSDoc documentation to better explain the rationale
behind each function, its input parameters and outputs.
---
 scripts/deleteAndLoadSnapsets.js | 227 ++++++++++++++++++++-----------
 1 file changed, 150 insertions(+), 77 deletions(-)

diff --git a/scripts/deleteAndLoadSnapsets.js b/scripts/deleteAndLoadSnapsets.js
index 10ce6785e..dd4b1fd6c 100644
--- a/scripts/deleteAndLoadSnapsets.js
+++ b/scripts/deleteAndLoadSnapsets.js
@@ -52,11 +52,8 @@ gpii.dataLoader.initOptions = function (processArgv) {
     dbOptions.couchDbUrl = processArgv[2];
     dbOptions.staticDataDir = processArgv[3];
     dbOptions.buildDataDir = processArgv[4];
-    if (processArgv.length > 5 && processArgv[5] === "--justDelete") { // for debugging.
-        dbOptions.justDelete = true;
-    } else {
-        dbOptions.justDelete = false;
-    }
+    // for debugging
+    dbOptions.justDelete = processArgv.length > 5 && processArgv[5] === "--justDelete";
 
     // Set up database specific options
     dbOptions.viewsUrl = dbOptions.couchDbUrl + "/_design/views";
@@ -92,12 +89,15 @@ gpii.dataLoader.initOptions = function (processArgv) {
     return dbOptions;
 };
 
-/*
- * Loads the static data from disk, and creates a separate reference to the
- * views document.
- * @param {Object} options - Object that has the path to the directory
- *                           containing the static data. On output, contains
- *                           members for the new views and other static data.
+/**
+ * Reads the static data from disk, and creates a separate reference to the
+ * views document.  Two fields are added to the `options` parameter on return:
+ *   options.staticData {Array} - Array of Objects to be put in the database.
+ *   options.newViews {Object} - The data used to update the `_design/views`
+ *                               record.
+ * @param {Object} options - The source of the static data:
+ * @param {String} options.staticDataDir - The path to the directory containing
+ *                                         the static data.
  */
 gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     var data = gpii.dataLoader.getDataFromDirectory(options.staticDataDir);
@@ -111,17 +111,64 @@ gpii.dataLoader.loadStaticDataFromDisk = function (options) {
     options.staticData = data;
     options.newViews = views;
     fluid.log("Retrieved static data from: '" + options.staticDataDir + "'");
-    fluid.log("\tViews data " + ( views ? "retrieved." : "missing." ));
+    fluid.log("\tViews data " + ( views ? "read." : "missing." ));
 };
 
-/*
- * Utility to configure a step:  creates a response callback, binds it to an
+/**
+ * Response handler function used for the callback argument of an
+ * {http.ClientRequest}.
+ * @callback ResponseCallback
+ * @param {http.IncomingMessage} response - Response object.
+ */
+
+/**
+ * Function that processes the data passed via the {ResponseCallback}.
+ * @callback ResponseDataHandler
+ * @param {String} responseString - The raw response data.
+ * @param {Object} options - Other information used by this handler; documented
+ *                           by specific data handler functions.
+ */
+
+/**
+ * POST request headers.
+ * @typedef {Object} PostRequestHeaders
+ * @property {String} Accept - "application/json" (constant).
+ * @property {String} Content-Length - Computed and filled in per request (variable).
+ * @property {String} Content-Type - "application/json" (constant).
+ */
+
+/**
+ * The POST request options for bulk updates.
+ * @typedef {Object} PostRequestOptions
+ * @property {String} hostname - The database host name (constant).
+ * @property {String} port - The port associated with the URL (constant).
+ * @property {String} path - The bulk documents command: "/gpii/_bulk_docs" (constant).
+ * @property {String} auth - Authorization for access (constant).
+ * @property {String} method - "POST" (constant).
+ * @property {PostRequestHeaders} headers - The POST headers.
+ */
+
+/**
+ * Utility to configure a step:  Creates a response callback, binds it to an
  * http database request, and configures a promise to resolve/reject when the
  * response callback finishes or fails.
- * @param {Object} details - Specific information for the request and response,
- *                           such as error messages, urls, POST data, and so on.
- *                           These details are set appropriately by the caller.
- * @param {Object} options - Additional options.
+ * @param {Object} details - Specific information for the request and response.
+ *                           These details are set appropriately by the caller:
+ * @param {String} details.requestErrMsg - Error message to display on a request
+ *                                         error.
+ * @param {ResponseDataHandler} details.responseDataHandler -
+ *                                  Function for processing the data returned in
+ *                                  the response.
+ * @param {String} details.responseErrMsg - Error message to display on a
+ *                                          response error.
+ * @param {Array} details.dataToPost - Optional: if present, a POST request is
+ *                                     used.
+ * @param {String} details.requestUrl - If not a POST request, the URL for a GET
+ *                                      request.
+ * @param {Object} options - Post request:
+ * @param {PostRequestOptions} options.postOptions - If a POST request is used,
+ *                                                   contains the specifics of
+ *                                                   the request.
  * @return {Promise} - A promise that resolves the configured step.
  */
 gpii.dataLoader.configureStep = function (details, options) {
@@ -146,9 +193,10 @@ gpii.dataLoader.configureStep = function (details, options) {
     return togo;
 };
 
-/*
+/**
  * Create the step that loads the static data into the database.
- * @param {Object} options - Object that has the static data to load.
+ * @param {Object} options - The static data:
+ * @param {Array} options.staticData - The static data to load.
  * @return {Promise} - A promise that resolves loading the static data.
  */
 gpii.dataLoader.createStaticDataStep = function (options) {
@@ -163,9 +211,10 @@ gpii.dataLoader.createStaticDataStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
+/**
  * Create the step that retrieves the current views from the database.
- * @param {Object} options - Object containing the views URL into the database.
+ * @param {Object} options - The URL to query the database with:
+ * @param {String} options.viewsUrl - The `_design/views` URL.
  * @return {Promise} - A promise that resolves retrieving the old views.
  */
 gpii.dataLoader.createFetchOldViewsStep = function (options) {
@@ -183,10 +232,14 @@ gpii.dataLoader.createFetchOldViewsStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
- * Create the step that updates the views in the database.
- * @param {Object} options - New views data to update with, and the old views
- *                           currently in the database.
+/**
+ * Create the step that updates the views in the database.  If the new views
+ * are the same as the old views, the `_design/views` record is not updated.
+ * @param {Object} options - New and old `_design/views` data:
+ * @param {Array} options.oldViews - the old views currently in the database.
+ *                                   Its `id` and `rev` fields are needed for
+ *                                   any update.
+ * @param {Object} options.newViews - the new views data to update with.
  * @return {Promise} - A promise that resolves updating the views.
  */
 gpii.dataLoader.createUpdateViewsStep = function (options) {
@@ -217,12 +270,13 @@ gpii.dataLoader.createUpdateViewsStep = function (options) {
 };
 
 /**
- * Find the Prefs Safes of type "snapset", mark them to be deleted and add
+ * Find the Prefs Safes of type "snapset", mark them to be deleted, and add
  * them to an array of records to remove.
- * @param {String} responseString - The response from the database query for
- *                                  retrieving the snapset PrefsSafes records
- * @param {Object} options - Data loader options used to store the current set
- *                           of processed snapsets.
+ * @param {String} responseString - The response from the database query -- the
+ *                                  retrieved snapset PrefsSafes records.
+ * @param {Object} options - Used to store the snapsets:
+ * @param {Array} options.snapsetPrefsSafes - On output, contains the "snapset"
+ *                                            PrefsSafes marked for deletion.
  * @return {Array} - The snapset PrefsSafes records marked for deletion.
  */
 gpii.dataLoader.processSnapsets = function (responseString, options) {
@@ -239,10 +293,14 @@ gpii.dataLoader.processSnapsets = function (responseString, options) {
 /**
  * Find the GPII Key records that are associated with a snapset PrefsSafe, mark
  * them for deletion, and add them to array of records to delete.
- * @param {String} responseString - The response from the database query for
- *                                  retrieving all the GPII Keys.
- * @param {Object} options - Data loader options used to store the current set
- *                           of snapset GPII keys.
+ * @param {String} responseString - The response from the database query -- all
+ *                                  of the GPII Keys in the database.
+ * @param {Object} options - Used to find and store the snapset GPII Keys:
+ * @param {Array} options.snapsetPrefsSafes - Contains the relevant PrefsSafes
+ *                                            to use to find their associated
+ *                                            GPII Keys.
+ * @param {Array} options.gpiiKeys - On output, contains the snapset GPII Key
+ *                                   records marked for deletion.
  * @return {Array} - The GPII Key records marked for deletion.
  */
 gpii.dataLoader.processGpiiKeys = function (responseString, options) {
@@ -262,7 +320,8 @@ gpii.dataLoader.processGpiiKeys = function (responseString, options) {
  * @param {Array} gpiiKeyRecords - Array of GPII Key records from the database.
  * @param {Array} snapSets - Array of snapset Prefs Safes whose id references
  *                           its associated GPII Key record.
- * @return {Array} - the values from the gpiiKeyRecords that are snapset GPII Keys.
+ * @return {Array} - the values from the gpiiKeyRecords that are snapset GPII
+ *                   Keys, marked for deletion.
  */
 gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets) {
     var gpiiKeysToDelete = [];
@@ -282,11 +341,12 @@ gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, sn
 
 /**
  * Utility to wrap all the pieces to make a bulk documents deletion request
- * for the snapset Prefs Safes and their associated GPII keys.  Its intended use
- * is the parameter of the appropriate promise.then() call.
- * @param {Object} batchDeleteResponse - the reponse handler configured for the
- *                                       batch delete request.
- * @param {Object} options - Object that contains the records to be deleted.
+ * for the snapset Prefs Safes and their associated GPII keys.
+ * @param {ResponseCallback} batchDeleteResponse - Reponse handler for the batch
+ *                                                 delete request.
+ * @param {Object} options - The records to be deleted:
+ * @param {Array} options.snapsetPrefsSafes - "snapset" PrefsSafes to delete.
+ * @param {Array} options.gpiiKeys - Associated GPII Keys to delete.
  * @return {http.ClientRequest} - The http request object.
  */
 gpii.dataLoader.configureBatchDelete = function (batchDeleteResponse, options) {
@@ -297,10 +357,12 @@ gpii.dataLoader.configureBatchDelete = function (batchDeleteResponse, options) {
 };
 
 /**
- * Create a function that makes a bulk docs POST request using the given data.
+ * Create an http request for a bulk docs POST request using the given data.
  * @param {Object} dataToPost - JSON data to POST and process in bulk.
- * @param {Object} responseHandler - http response handler for the request.
- * @param {Object} options - Data loader options, specifically the POST options.
+ * @param {ResponseCallback} responseHandler - http response callback for the
+ *                                             request.
+ * @param {Object} options - Post request options:
+ * @param {PostRequestOptions} options.postOptions - the POST request specifics.
  * @return {http.ClientRequest} - An http request object.
  */
 gpii.dataLoader.createPostRequest = function (dataToPost, responseHandler, options) {
@@ -314,14 +376,16 @@ gpii.dataLoader.createPostRequest = function (dataToPost, responseHandler, optio
 /**
  * Generate a response handler, setting up the given promise to resolve/reject
  * at the correct time.
- * @param {Function} handleEnd - Function to call that deals with the response
- *                               data when the response receives an "end" event.
- * @param {Object} options - Data loader options passed to handleEnd().
+ * @param {ResponseDataHandler} handleEnd - Function that processes the response
+ *                                          data when the response receives an
+ *                                          "end" event.
+ * @param {Object} options - Data loader options passed to `handleEnd()`.
  * @param {Promise} promise - Promise to resolve/reject on a response "end" or
  *                           "error" event.
  * @param {String} errorMsg - Optional error message to prepend to the error
  *                            received from a response "error" event.
- * @return {Function} - Function reponse callback for an http request.
+ * @return {ResponseCallback} - Reponse callback function suitable for an http
+ *                              request.
  */
 gpii.dataLoader.createResponseHandler = function (handleEnd, options, promise, errorMsg) {
     if (!errorMsg) {
@@ -365,8 +429,8 @@ gpii.dataLoader.createResponseHandler = function (handleEnd, options, promise, e
  * return.  It is up to the caller to trigger the request by calling its end()
  * function.
  * @param {String} databaseURL - URL to query the database with.
- * @param {Function} handleResponse - callback that processes the response from
- *                                    the request.
+ * @param {ResponseCallback} handleResponse - callback that processes the
+ *                                            response from the request.
  * @param {String} errorMsg - optional error message for request errors.
  * @return {http.ClientRequest} - The http request object.
  */
@@ -379,7 +443,7 @@ gpii.dataLoader.queryDatabase = function (databaseURL, handleResponse, errorMsg)
 };
 
 /**
- * Get all the json files from the given directory, then loop to put their
+ * Read all the json files from the given directory, then loop to put their
  * contents into an array of Objects.
  * @param {String} dataDir - Directory containing the files to load.
  * @return {Array} - Each element of the array is an Object based on the
@@ -397,12 +461,13 @@ gpii.dataLoader.getDataFromDirectory = function (dataDir) {
     return contentArray;
 };
 
-/*
- * Create the step that fetches the current snapset Prefs Safes from the
+/**
+ * Create the step that fetches the current "snapset" Prefs Safes from the
  * database.
- * @param {Object} options - Object that has the view for finding snap set
- *                           Prefs Safes records in the database.
- * @return {Promise} - A promise that resolves to the set of snapset PrefsSafes
+ * @param {Object} options - Object for querying the database:
+ * @param {String} options.prefsSafesViewUrl - Views URL for finding all the
+ *                                             "snapset" PrefsSafes records.
+ * @return {Promise} - A promise that resolves to the set of "snapset" PrefsSafes
  *                     currently in the database.
  */
 gpii.dataLoader.createFetchSnapsetsStep = function (options) {
@@ -415,13 +480,14 @@ gpii.dataLoader.createFetchSnapsetsStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
+/**
  * Create the step that fetches the current GPII keys associate with the snapset
  * Prefs Safes.
- * @param {Object} options - Object that has the view for finding all GPII Key
- *                           records in the database.
+ * @param {Object} options - Object for querying the database:
+ * @param {String} options.gpiiKeysViewUrl - Views URL for finding all GPII Key
+ *                                           records in the database.
  * @return {Promise} - A promise that resolves to the set of GPII keys in the
- *                     database that correspond to snapset PrefsSafes.
+ *                     database that correspond to "snapset" PrefsSafes.
  */
 gpii.dataLoader.createFetchGpiiKeysStep = function (options) {
     var details = {
@@ -433,12 +499,15 @@ gpii.dataLoader.createFetchGpiiKeysStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
+/**
  * Log how many snapset Prefs Safes and GPII Keys were deleted.
  * @param {String} responseString - Response from the database (ignored)
- * @param {Object} options - Object that contains the sets of Prefs Safes and
- *                           their keys.
- * @return {Object} - The number of snapsets and gpiiKeys deleted.
+ * @param {Object} options - Object containing the sets of Prefs Safes and
+ *                           their GPII keys:
+ * @param {Array} options.snapsetPrefsSafes - The set of Prefs Safes.
+ * @param {Array} options.gpiiKeys - The set of associated GPII Keys.
+ * @return {Object} - An object with properties "snapsets" and "gpiiKeys" that
+ *                    are the number of snapsets and gpiiKeys deleted.
  */
 gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
     fluid.log(  "Deleted " +
@@ -451,12 +520,13 @@ gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
     };
 };
 
-/*
+/**
  * Create the step that deletes, in batch, the current snapset Prefs Safes and
  * their associated GPII keys.
- * @param {Promise} previousStep - Promise from a previous step whose fulfillment
- *                                 triggers the bulk delete request.
- * @param {Object} options - Object that contains the records to be deleted.
+ * @param {Object} options - The records to be deleted:
+ * @param {Array} options.snapsetPrefsSafes - The "snapset" PrefsSafe records to
+ *                                            delete.
+ * @param {Array} options.gpiiKeys - The GPII Key records to delete.
  * @return {Promise} - The promise that resolves the deletion.
  */
 gpii.dataLoader.createBatchDeleteStep = function (options) {
@@ -467,22 +537,25 @@ gpii.dataLoader.createBatchDeleteStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
- * Log the uploading of all the snapset and user Prefs Safes.
+/**
+ * Log the uploading of all the "snapset" Prefs Safes and their GPII Keys.
  * @param {String} responseString - Response from the database (ignored)
- * @param {Object} options - Object that contains the directories from which
- *                           the Prefs Safes were loaded.
+ * @param {Object} options - The directory containing the data:
+ * @param {String} options.buildDataDir - The directory from which the Prefs
+ *                                        Safes and GPII keys were loaded.
+ * @return {String} - A message to indicate that the upload is complete.
  */
 gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
     fluid.log("Bulk loading of build data from '" + options.buildDataDir + "'");
     return "Uploaded latest snapsets preferences";
 };
 
-/*
- * Create the step that uploads, in batch, the new snapset Prefs Safes, and
+/**
+ * Create the step that uploads, in batch, the new "snapset" Prefs Safes, and
  * their associated GPII keys.
- * @param {Object} options - Object that has the paths to the directory that
- *                           contains the latest snapsets.
+ * @param {Object} options - The directory containing the data:
+ * @param {String} options.buildDataDir - The directory from which to load the
+ *                                        Prefs Safes and GPII keys data.
  * @return {Promise} - A promise that resolves the upload.
  */
 gpii.dataLoader.createBatchUploadStep = function (options) {
@@ -494,7 +567,7 @@ gpii.dataLoader.createBatchUploadStep = function (options) {
     return gpii.dataLoader.configureStep(details, options);
 };
 
-/*
+/**
  * Create and execute the steps to update the database.
  */
 gpii.dataLoader.orchestrate = function () {

From d9ac25e3d3e9a763406d3fae375fe616b1079c23 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Mon, 15 Oct 2018 17:17:45 -0400
Subject: [PATCH 40/41] GPII-3138:  Update snapsets in the database

- modified the main README.md to better explain the various ways
that the source preferences files are converted into PrefsSafes
and GPII Keys, and for which purposes they are used.
- minor grammaticial changes to README.txt.
---
 README.md                  | 42 ++++++++++++++++++++++----------------
 testData/dbData/README.txt |  2 +-
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 5af77b6cb..b7ebc9714 100644
--- a/README.md
+++ b/README.md
@@ -98,24 +98,30 @@ have set the `NODE_ENV` variable.
 
 ### Convert Preferences Data
 
-GPII has two sets of preferences JSON5 data files, located at `%gpii-universal/testData/preferences` and `%gpii-universal/tests/data/preferences`:
-
-* The preferences files for running GPII and for integration tests are located at
-  `%gpii-universal/testData/preferences`.  These files are converted into two types of preferences safes and GPII keys:
-  * They are converted into `snapset` preferences safes and GPII keys and placed into the
-    `%gpii-universal/build/dbData/snapset/` folder.   These are used to update the snapsets in CouchDB when GPII is
-    run in a production or staging configuration.
-  * They are also converted into `user` preferences safes and GPII keys and placed into the
-    `%gpii-universal/build/dbData/user/` folder.   These are used with PouchDB when GPII runs in a development
-    configuration or when running GPII integration tests.
-* The preferences files for running node tests are located at `%gpii-universal/tests/data/preferences`.  These files
-  are converted into preferences safes and GPII keys:
-  * They are converted into `user` preferences safes and GPII keys and placed into the
-    `%gpii-universal/build/tests/dbData/user` folder.   These are used during testing with PouchDB.
-
-When any preferences file in either one of these two directories are modified, running `npm run postinstall` will generate
-gpiiKeys.json and prefsSafes.json, the files that are in the structure to be loaded into PouchDB/CouchDB, based off
-these directories. This step is needed for the modification to be applied to GPII.
+GPII has two sets of source preferences JSON5 data files, located at `%gpii-universal/testData/preferences` and `%gpii-universal/tests/data/preferences/`.  These are converted and used in various configurations.
+
+The preferences files in `%gpii-universaluniversal/testData/preferences/` are converted into both `snapset` and `user` preferences:
+* `%gpii-universal/build/dbData/snapset/gpiiKeys.json`
+* `%gpii-universal/build/dbData/snapset/prefsSafes.json`
+* `%gpii-universal/build/dbData/user/gpiiKeys.json`
+* `%gpii-universal/build/dbData/user/prefsSafes.json`
+
+The above `snapset` preferences safes and GPII keys are:
+1. loaded into the production and the staging CouchDB in cloud environments.
+2. loaded into the PouchDB when GPII runs locally, regardless of which configuration is used.
+
+The above `user` preferences are loaded into the local PouchDB for running GPII integration tests.
+
+The preferences in `%gpii-universal/tests/data/preferences/` are converted into `user` preferences:
+* `%gpii-universal/build/tests/dbData/user/gpiiKeys.json`
+* `%gpii-universal/build/tests/dbData/user/prefsSafes.json`
+
+These `user` preferences and the above `snapset` preferences are used with a PouchDB when GPII runs in a development configuration.
+
+When any preferences file in either one of the two source directories (`%gpii-universaluniversal/testData/preferences/`
+or `%gpii-universal/tests/data/preferences/`) are modified, running `npm run postinstall` will generate
+gpiiKeys.json and prefsSafes.json files, whose contents are structured for loading into PouchDB/CouchDB.
+This step is needed for any preferences modifications that are to be applied to GPII.
 
 ### Running browser tests
 
diff --git a/testData/dbData/README.txt b/testData/dbData/README.txt
index 1a9b26744..29d6b6ea9 100644
--- a/testData/dbData/README.txt
+++ b/testData/dbData/README.txt
@@ -8,7 +8,7 @@ Each data file corresponds to a document structure specified in the [GPII Data M
 * gpiiAppInstallationClients.json: Contains the information of all registered GPII app installations.
 * views.json: The PouchDB/CouchDB "_design/views" functions.
 
-Additional preferences DB data is found elsewhere and used in conjunction with the files in this folder.  The preferences files located in %universal/testData/preferences/ are converted into both "snapset" and "user" preferences.  The following "snapset" folder contain snapset PrefsSafes and GPII keys for (1) running GPII in production and staging GPII configurations where the data is loaded into CouchDB, and (2) running GPII in its development configuration where the data is loaded into the local PouchDB:
+Additional preferences DB data are found elsewhere and used in conjunction with the files in this folder.  The preferences files located in %universal/testData/preferences/ are converted into both "snapset" and "user" preferences.  The following "snapset" folder contains snapset PrefsSafes and GPII keys for (1) running GPII in production and staging GPII configurations where the data is loaded into CouchDB, and (2) running GPII in its development configuration where the data is loaded into the local PouchDB:
 * %universal/build/dbData/snapset/gpiiKeys.json
 * %universal/build/dbData/snapset/prefsSafes.json
 

From ebde69dbd58550aae0d4ab3a0b779a839e0b2ce8 Mon Sep 17 00:00:00 2001
From: Joseph Scheuhammer <clown@alum.mit.edu>
Date: Wed, 17 Oct 2018 10:21:51 -0400
Subject: [PATCH 41/41] GPII-3138:  Update snapsets in the database

Fixed markdown errors.
---
 README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b7ebc9714..497c22438 100644
--- a/README.md
+++ b/README.md
@@ -98,21 +98,26 @@ have set the `NODE_ENV` variable.
 
 ### Convert Preferences Data
 
-GPII has two sets of source preferences JSON5 data files, located at `%gpii-universal/testData/preferences` and `%gpii-universal/tests/data/preferences/`.  These are converted and used in various configurations.
+GPII has two sets of source preferences JSON5 data files, located at `%gpii-universal/testData/preferences` and
+`%gpii-universal/tests/data/preferences/`.  These are converted and used in various configurations.
+
+The preferences files in `%gpii-universaluniversal/testData/preferences/` are converted into both `snapset` and
+`user` preferences:
 
-The preferences files in `%gpii-universaluniversal/testData/preferences/` are converted into both `snapset` and `user` preferences:
 * `%gpii-universal/build/dbData/snapset/gpiiKeys.json`
 * `%gpii-universal/build/dbData/snapset/prefsSafes.json`
 * `%gpii-universal/build/dbData/user/gpiiKeys.json`
 * `%gpii-universal/build/dbData/user/prefsSafes.json`
 
 The above `snapset` preferences safes and GPII keys are:
+
 1. loaded into the production and the staging CouchDB in cloud environments.
 2. loaded into the PouchDB when GPII runs locally, regardless of which configuration is used.
 
 The above `user` preferences are loaded into the local PouchDB for running GPII integration tests.
 
 The preferences in `%gpii-universal/tests/data/preferences/` are converted into `user` preferences:
+
 * `%gpii-universal/build/tests/dbData/user/gpiiKeys.json`
 * `%gpii-universal/build/tests/dbData/user/prefsSafes.json`