Merge pull request #47 from jeffposnick/offline-analytics

Add in support for queueing and retrying failed Google Analytics pings.

app/scripts/shed-offline-analytics.js

@@ -0,0 +1,72 @@
+// TODO (jeffposnick): Reach out to inexorabletash (Joshua Bell) about making this into a package
+// that we can install in a standard location and add to the dependencies.
+importScripts('scripts/third_party/simpledb_polyfill.js');
+
+var DB_NAME = 'shed-offline-analytics';
+var EXPIRATION_TIME_DELTA = 86400000; // One day, in milliseconds.
+
+function replayQueuedRequests() {
+  simpleDB.open(DB_NAME).then(function(db) {
+    db.forEach(function(url) {
+      db.get(url).then(function(originalTimestamp) {
+        var timeDelta = Date.now() - originalTimestamp;
+        // See https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#qt
+        var replayUrl = url + '&qt=' + timeDelta;
+
+        console.log('About to replay:', replayUrl);
+        fetch(replayUrl).then(function(response) {
+          if (response.status >= 500) {
+            // This will cause the promise to reject, triggering the .catch() function.
+            return Response.error();
+          }
+
+          console.log('Replay succeeded:', replayUrl);
+          db.delete(url);
+        }).catch(function(error) {
+          if (timeDelta > EXPIRATION_TIME_DELTA) {
+            // After a while, Google Analytics will no longer accept an old ping with a qt=
+            // parameter. The advertised time is ~4 hours, but we'll attempt to resend up to 24
+            // hours. This logic also prevents the requests from being queued indefinitely.
+            console.error('Replay failed, but the original request is too old to retry any further. Error:', error);
+            db.delete(url);
+          } else {
+            console.error('Replay failed, and will be retried the next time the service worker starts. Error:', error);
+          }
+        });
+      });
+    });
+  });
+}
+
+function queueFailedRequest(request) {
+  console.log('Queueing failed request:', request);
+
+  simpleDB.open(DB_NAME).then(function(db) {
+    db.set(request.url, Date.now());
+  });
+}
+
+function handleAnalyticsCollectionRequest(request) {
+  return fetch(request).then(function(response) {
+    if (response.status >= 500) {
+      // This will cause the promise to reject, triggering the .catch() function.
+      // It will also result in a generic HTTP error being returned to the controlled page.
+      return Response.error();
+    } else {
+      return response;
+    }
+  }).catch(function() {
+    queueFailedRequest(request);
+  });
+}
+
+// TODO (jeffposnick): Is there any way to use wildcards for the protocols and domains?
+// TODO (jeffposnick): See if there's a way to avoid :ignored. See https://github.com/wibblymat/shed/issues/17
+shed.router.get('http://www.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest);
+shed.router.get('https://www.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest);
+shed.router.get('https://ssl.google-analytics.com/collect?:ignored', handleAnalyticsCollectionRequest);
+
+shed.router.get('http://www.google-analytics.com/analytics.js', shed.networkFirst);
+shed.router.get('https://www.google-analytics.com/analytics.js', shed.networkFirst);
+
+replayQueuedRequests();

app/scripts/third_party/shed.js

@@ -28,14 +28,14 @@ var Route = function(method, path, handler) {
 };
 
 Route.prototype.makeHandler = function(url) {
-    var match = this.regexp.exec(url);
-    var values = {};
-    this.keys.forEach(function(key, index) {
-      values[key.name] = match[index + 1];
-    });
-    return function(request) {
-      return this.handler(request, values);
-    }.bind(this);
+  var match = this.regexp.exec(url);
+  var values = {};
+  this.keys.forEach(function(key, index) {
+    values[key.name] = match[index + 1];
+  });
+  return function(request) {
+    return this.handler(request, values);
+  }.bind(this);
 };
 
 var Router = function() {
@@ -68,8 +68,10 @@ Router.prototype.any = function(path, handler) {
 };
 
 Router.prototype.add = function(method, path, handler) {
+  method = method.toLowerCase();
+  var route = new Route(method, path, handler);
   this.routes[method] = this.routes[method] || {};
-  this.routes[method][path] = new Route(method.toLowerCase(), path, handler);
+  this.routes[method][route.regexp.toString()] = route;
 };
 
 Router.prototype.matchMethod = function(method, url) {
@@ -107,6 +109,11 @@ var cacheName = cachePrefix + version;
 var preCacheItems = [];
 var DEBUG = false;
 
+// A regular expression to apply to HTTP response codes. Codes that match will
+// be considered successes, while others will not, and will not be cached.
+// TODO: Make this user configurable
+var SUCCESS_RESPONSES = /^0|([123]\d\d)|(40[14567])|410$/;
+
 // Internal Helpers
 
 function debug(message) {
@@ -131,9 +138,13 @@ function cacheFetch(request) {
 
 function fetchAndCache(request) {
   return networkFetch(request.clone()).then(function(response) {
-    openCache().then(function(cache) {
-      cache.put(request, response);
-    });
+
+    // Only cache successful responses
+    if (SUCCESS_RESPONSES.test(response.status)) {
+      openCache().then(function(cache) {
+        cache.put(request, response);
+      });
+    }
 
     return response.clone();
   });
@@ -214,8 +225,25 @@ function networkOnly(request) {
 
 function networkFirst(request) {
   debug('Trying network first');
-  return fetchAndCache(request).catch(function(error) {
-    debug('Cache fallback');
+  return fetchAndCache(request).then(function(response) {
+    if (SUCCESS_RESPONSES.test(response.status)) {
+      return response;
+    }
+
+    return cacheFetch(request).then(function(cacheResponse) {
+      debug('Response was an HTTP error');
+      if (cacheResponse) {
+        debug('Resolving with cached response instead');
+        return cacheResponse;
+      } else {
+        // If we didn't have anything in the cache, it's better to return the
+        // error page than to return nothing
+        debug('No cached result, resolving with HTTP error response from network');
+        return response;
+      }
+    });
+  }).catch(function(error) {
+    debug('Network error, fallback to cache');
     return cacheFetch(request);
   });
 }
@@ -232,7 +260,7 @@ function cacheFirst(request) {
       return response;
     }
 
-    return networkFetch(request);
+    return fetchAndCache(request);
   });
 }
 
@@ -241,9 +269,9 @@ function fastest(request) {
   var reasons = [];
 
   var maybeReject = function(reason) {
-    reasons.push(reason);
+    reasons.push(reason.toString());
     if (rejected) {
-      return Promise.reject(reasons);
+      return Promise.reject(new Error('Both cache and network failed: "' + reasons.join('", "') + '"'));
     }
     rejected = true;
   };
@@ -530,11 +558,11 @@ if (!CacheStorage.prototype.match) {
       return cacheNames.reduce(function(chain, cacheName) {
         return chain.then(function() {
           return match || caches.open(cacheName).then(function(cache) {
-            return cache.match(request, opts);
-          }).then(function(response) {
-            match = response;
-            return match;
-          });
+              return cache.match(request, opts);
+            }).then(function(response) {
+              match = response;
+              return match;
+            });
         });
       }, Promise.resolve());
     });

app/scripts/third_party/simpledb_polyfill.js

@@ -0,0 +1,157 @@
+// From https://gist.github.com/inexorabletash/c8069c042b734519680c (Joshua Bell)
+
+(function(global) {
+  var SECRET = Object.create(null);
+  var DB_PREFIX = '$SimpleDB$';
+  var STORE = 'store';
+
+  function SimpleDBFactory(secret) {
+    if (secret !== SECRET) throw TypeError('Invalid constructor');
+  }
+  SimpleDBFactory.prototype = {
+    open: function(name) {
+      return new Promise(function(resolve, reject) {
+        var request = indexedDB.open(DB_PREFIX + name);
+        request.onupgradeneeded = function() {
+          var db = request.result;
+          db.createObjectStore(STORE);
+        };
+        request.onsuccess = function() {
+          var db = request.result;
+          resolve(new SimpleDB(SECRET, name, db));
+        };
+        request.onerror = function() {
+          reject(request.error);
+        };
+      });
+    },
+    delete: function(name) {
+      return new Promise(function(resolve, reject) {
+        var request = indexedDB.deleteDatabase(DB_PREFIX + name);
+        request.onsuccess = function() {
+          resolve(undefined);
+        };
+        request.onerror = function() {
+          reject(request.error);
+        };
+      });
+    }
+  };
+
+  function SimpleDB(secret, name, db) {
+    if (secret !== SECRET) throw TypeError('Invalid constructor');
+    this._name = name;
+    this._db = db;
+  }
+  SimpleDB.cmp = indexedDB.cmp;
+  SimpleDB.prototype = {
+    get name() {
+      return this._name;
+    },
+    get: function(key) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var req = store.get(key);
+        // NOTE: Could also use req.onsuccess/onerror
+        tx.oncomplete = function() { resolve(req.result); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    set: function(key, value) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var req = store.put(value, key);
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    delete: function(key) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var req = store.delete(key);
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    clear: function() {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var request = store.clear();
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    forEach: function(callback, options) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        options = options || {};
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var request = store.openCursor(
+          options.range,
+          options.direction === 'reverse' ? 'prev' : 'next');
+        request.onsuccess = function() {
+          var cursor = request.result;
+          if (!cursor) return;
+          try {
+            var terminate = callback(cursor.key, cursor.value);
+            if (!terminate) cursor.continue();
+          } catch (ex) {
+            tx.abort(); // ???
+          }
+        };
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    getMany: function(keys) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        var results = [];
+        for (var key of keys) {
+          store.get(key).onsuccess(function(result) {
+            results.push(result);
+          });
+        }
+        tx.oncomplete = function() { resolve(results); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    setMany: function(entries) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        for (var entry of entries) {
+          store.put(entry.value, entry.key);
+        }
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    },
+    deleteMany: function(keys) {
+      var that = this;
+      return new Promise(function(resolve, reject) {
+        var tx = that._db.transaction(STORE, 'readwrite');
+        var store = tx.objectStore(STORE);
+        for (var key of keys)
+        store.delete(key);
+        tx.oncomplete = function() { resolve(undefined); };
+        tx.onabort = function() { reject(tx.error); };
+      });
+    }
+  };
+
+  global.simpleDB = new SimpleDBFactory(SECRET);
+  global.SimpleDBKeyRange = IDBKeyRange;
+}(self));

app/sw.js

@@ -15,6 +15,7 @@
  */
 
 importScripts('scripts/third_party/shed.js');
+importScripts('scripts/shed-offline-analytics.js');
 
 shed.precache(['/temporary_api/precache/test.html']);