From f788089695cb4def98ab555d0ddf5767811bb0b1 Mon Sep 17 00:00:00 2001 From: Jeff Posnick Date: Sat, 20 Dec 2014 21:07:23 -0500 Subject: [PATCH] Add in support for queueing and retrying failed Google Analytics pings. --- app/scripts/shed-offline-analytics.js | 72 +++++++++ app/scripts/third_party/shed.js | 72 ++++++--- app/scripts/third_party/simpledb_polyfill.js | 157 +++++++++++++++++++ app/sw.js | 1 + 4 files changed, 280 insertions(+), 22 deletions(-) create mode 100644 app/scripts/shed-offline-analytics.js create mode 100644 app/scripts/third_party/simpledb_polyfill.js diff --git a/app/scripts/shed-offline-analytics.js b/app/scripts/shed-offline-analytics.js new file mode 100644 index 00000000..e2bab685 --- /dev/null +++ b/app/scripts/shed-offline-analytics.js @@ -0,0 +1,72 @@ +// TODO (jeffposnick): Reach out to inexorabletash (Joshua Bell) about making this into a package +// that we can install in a standard location and add to the dependencies. +importScripts('scripts/third_party/simpledb_polyfill.js'); + +var DB_NAME = 'shed-offline-analytics'; +var EXPIRATION_TIME_DELTA = 86400000; // One day, in milliseconds. + +function replayQueuedRequests() { + simpleDB.open(DB_NAME).then(function(db) { + db.forEach(function(url) { + db.get(url).then(function(originalTimestamp) { + var timeDelta = Date.now() - originalTimestamp; + // See https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#qt + var replayUrl = url + '&qt=' + timeDelta; + + console.log('About to replay:', replayUrl); + fetch(replayUrl).then(function(response) { + if (response.status >= 500) { + // This will cause the promise to reject, triggering the .catch() function. + return Response.error(); + } + + console.log('Replay succeeded:', replayUrl); + db.delete(url); + }).catch(function(error) { + if (timeDelta > EXPIRATION_TIME_DELTA) { + // After a while, Google Analytics will no longer accept an old ping with a qt= + // parameter. The advertised time is ~4 hours, but we'll attempt to resend up to 24 + // hours. This logic also prevents the requests from being queued indefinitely. + console.error('Replay failed, but the original request is too old to retry any further. Error:', error); + db.delete(url); + } else { + console.error('Replay failed, and will be retried the next time the service worker starts. Error:', error); + } + }); + }); + }); + }); +} + +function queueFailedRequest(request) { + console.log('Queueing failed request:', request); + + simpleDB.open(DB_NAME).then(function(db) { + db.set(request.url, Date.now()); + }); +} + +function handleAnalyticsCollectionRequest(request) { + return fetch(request).then(function(response) { + if (response.status >= 500) { + // This will cause the promise to reject, triggering the .catch() function. + // It will also result in a generic HTTP error being returned to the controlled page. + return Response.error(); + } else { + return response; + } + }).catch(function() { + queueFailedRequest(request); + }); +} + +// TODO (jeffposnick): Is there any way to use wildcards for the protocols and domains? +// TODO (jeffposnick): See if there's a way to avoid :ignored. See https://github.com/wibblymat/shed/issues/17 +shed.router.get('http://www.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest); +shed.router.get('https://www.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest); +shed.router.get('https://ssl.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest); + +shed.router.get('http://www.google-analytics.com/analytics.js', shed.networkFirst); +shed.router.get('https://www.google-analytics.com/analytics.js', shed.networkFirst); + +replayQueuedRequests(); diff --git a/app/scripts/third_party/shed.js b/app/scripts/third_party/shed.js index fc05d92a..f565d1a1 100644 --- a/app/scripts/third_party/shed.js +++ b/app/scripts/third_party/shed.js @@ -28,14 +28,14 @@ var Route = function(method, path, handler) { }; Route.prototype.makeHandler = function(url) { - var match = this.regexp.exec(url); - var values = {}; - this.keys.forEach(function(key, index) { - values[key.name] = match[index + 1]; - }); - return function(request) { - return this.handler(request, values); - }.bind(this); + var match = this.regexp.exec(url); + var values = {}; + this.keys.forEach(function(key, index) { + values[key.name] = match[index + 1]; + }); + return function(request) { + return this.handler(request, values); + }.bind(this); }; var Router = function() { @@ -68,8 +68,10 @@ Router.prototype.any = function(path, handler) { }; Router.prototype.add = function(method, path, handler) { + method = method.toLowerCase(); + var route = new Route(method, path, handler); this.routes[method] = this.routes[method] || {}; - this.routes[method][path] = new Route(method.toLowerCase(), path, handler); + this.routes[method][route.regexp.toString()] = route; }; Router.prototype.matchMethod = function(method, url) { @@ -107,6 +109,11 @@ var cacheName = cachePrefix + version; var preCacheItems = []; var DEBUG = false; +// A regular expression to apply to HTTP response codes. Codes that match will +// be considered successes, while others will not, and will not be cached. +// TODO: Make this user configurable +var SUCCESS_RESPONSES = /^0|([123]\d\d)|(40[14567])|410$/; + // Internal Helpers function debug(message) { @@ -131,9 +138,13 @@ function cacheFetch(request) { function fetchAndCache(request) { return networkFetch(request.clone()).then(function(response) { - openCache().then(function(cache) { - cache.put(request, response); - }); + + // Only cache successful responses + if (SUCCESS_RESPONSES.test(response.status)) { + openCache().then(function(cache) { + cache.put(request, response); + }); + } return response.clone(); }); @@ -214,8 +225,25 @@ function networkOnly(request) { function networkFirst(request) { debug('Trying network first'); - return fetchAndCache(request).catch(function(error) { - debug('Cache fallback'); + return fetchAndCache(request).then(function(response) { + if (SUCCESS_RESPONSES.test(response.status)) { + return response; + } + + return cacheFetch(request).then(function(cacheResponse) { + debug('Response was an HTTP error'); + if (cacheResponse) { + debug('Resolving with cached response instead'); + return cacheResponse; + } else { + // If we didn't have anything in the cache, it's better to return the + // error page than to return nothing + debug('No cached result, resolving with HTTP error response from network'); + return response; + } + }); + }).catch(function(error) { + debug('Network error, fallback to cache'); return cacheFetch(request); }); } @@ -232,7 +260,7 @@ function cacheFirst(request) { return response; } - return networkFetch(request); + return fetchAndCache(request); }); } @@ -241,9 +269,9 @@ function fastest(request) { var reasons = []; var maybeReject = function(reason) { - reasons.push(reason); + reasons.push(reason.toString()); if (rejected) { - return Promise.reject(reasons); + return Promise.reject(new Error('Both cache and network failed: "' + reasons.join('", "') + '"')); } rejected = true; }; @@ -530,11 +558,11 @@ if (!CacheStorage.prototype.match) { return cacheNames.reduce(function(chain, cacheName) { return chain.then(function() { return match || caches.open(cacheName).then(function(cache) { - return cache.match(request, opts); - }).then(function(response) { - match = response; - return match; - }); + return cache.match(request, opts); + }).then(function(response) { + match = response; + return match; + }); }); }, Promise.resolve()); }); diff --git a/app/scripts/third_party/simpledb_polyfill.js b/app/scripts/third_party/simpledb_polyfill.js new file mode 100644 index 00000000..3df10e9b --- /dev/null +++ b/app/scripts/third_party/simpledb_polyfill.js @@ -0,0 +1,157 @@ +// From https://gist.github.com/inexorabletash/c8069c042b734519680c (Joshua Bell) + +(function(global) { + var SECRET = Object.create(null); + var DB_PREFIX = '$SimpleDB$'; + var STORE = 'store'; + + function SimpleDBFactory(secret) { + if (secret !== SECRET) throw TypeError('Invalid constructor'); + } + SimpleDBFactory.prototype = { + open: function(name) { + return new Promise(function(resolve, reject) { + var request = indexedDB.open(DB_PREFIX + name); + request.onupgradeneeded = function() { + var db = request.result; + db.createObjectStore(STORE); + }; + request.onsuccess = function() { + var db = request.result; + resolve(new SimpleDB(SECRET, name, db)); + }; + request.onerror = function() { + reject(request.error); + }; + }); + }, + delete: function(name) { + return new Promise(function(resolve, reject) { + var request = indexedDB.deleteDatabase(DB_PREFIX + name); + request.onsuccess = function() { + resolve(undefined); + }; + request.onerror = function() { + reject(request.error); + }; + }); + } + }; + + function SimpleDB(secret, name, db) { + if (secret !== SECRET) throw TypeError('Invalid constructor'); + this._name = name; + this._db = db; + } + SimpleDB.cmp = indexedDB.cmp; + SimpleDB.prototype = { + get name() { + return this._name; + }, + get: function(key) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var req = store.get(key); + // NOTE: Could also use req.onsuccess/onerror + tx.oncomplete = function() { resolve(req.result); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + set: function(key, value) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var req = store.put(value, key); + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + delete: function(key) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var req = store.delete(key); + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + clear: function() { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var request = store.clear(); + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + forEach: function(callback, options) { + var that = this; + return new Promise(function(resolve, reject) { + options = options || {}; + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var request = store.openCursor( + options.range, + options.direction === 'reverse' ? 'prev' : 'next'); + request.onsuccess = function() { + var cursor = request.result; + if (!cursor) return; + try { + var terminate = callback(cursor.key, cursor.value); + if (!terminate) cursor.continue(); + } catch (ex) { + tx.abort(); // ??? + } + }; + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + getMany: function(keys) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + var results = []; + for (var key of keys) { + store.get(key).onsuccess(function(result) { + results.push(result); + }); + } + tx.oncomplete = function() { resolve(results); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + setMany: function(entries) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + for (var entry of entries) { + store.put(entry.value, entry.key); + } + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + }, + deleteMany: function(keys) { + var that = this; + return new Promise(function(resolve, reject) { + var tx = that._db.transaction(STORE, 'readwrite'); + var store = tx.objectStore(STORE); + for (var key of keys) + store.delete(key); + tx.oncomplete = function() { resolve(undefined); }; + tx.onabort = function() { reject(tx.error); }; + }); + } + }; + + global.simpleDB = new SimpleDBFactory(SECRET); + global.SimpleDBKeyRange = IDBKeyRange; +}(self)); diff --git a/app/sw.js b/app/sw.js index ee25b397..91e7d5e3 100644 --- a/app/sw.js +++ b/app/sw.js @@ -15,6 +15,7 @@ */ importScripts('scripts/third_party/shed.js'); +importScripts('scripts/shed-offline-analytics.js'); shed.precache(['/temporary_api/precache/test.html']);