🚢 v1.5.0

__New__:

- 📱 Support for mobile crawlers and bots;
- 🐞 Fix regression introduced in latest release affecting some of the
node.js setups;

__Changed__:

- 📋 Update documentation and examples;
- 📦 Updated NPM dependency `request-libcurl` to `v2.0.6`, *was
`2.0.3`*;
- 📦 Updated NPM [dev-dependency] `mocha` to `v7.1.0`, *was `6.2.2`*;
- 📦 Updated NPM [dev-dependency] `request` to `v2.88.2`, *was
`2.88.0`*;
- 📦 Updated NPM [dev-dependency] `underscore` to `v1.9.2`, *was
`1.9.0`*.

.versions

@@ -1,41 +1,41 @@
 allow-deny@1.1.0
-babel-compiler@7.4.0
-babel-runtime@1.4.0
+babel-compiler@7.5.2
+babel-runtime@1.5.0
 base64@1.0.12
 binary-heap@1.0.11
 boilerplate-generator@1.6.0
-callback-hook@1.2.0
+callback-hook@1.3.0
 check@1.3.1
 ddp@1.4.0
 ddp-client@2.3.3
 ddp-common@1.4.0
 ddp-server@2.3.0
 diff-sequence@1.1.1
 dynamic-import@0.5.1
-ecmascript@0.13.0
+ecmascript@0.14.2
 ecmascript-runtime@0.7.0
-ecmascript-runtime-client@0.9.0
-ecmascript-runtime-server@0.8.0
-ejson@1.1.0
+ecmascript-runtime-client@0.10.0
+ecmascript-runtime-server@0.9.0
+ejson@1.1.1
 fetch@0.1.1
 geojson-utils@1.0.10
 http@1.4.2
 id-map@1.1.0
 inter-process-messaging@0.1.0
-local-test:ostrio:spiderable-middleware@1.4.5
+local-test:ostrio:spiderable-middleware@1.5.0
 logging@1.1.20
 meteor@1.9.3
 minimongo@1.4.5
 modern-browsers@0.1.4
-modules@0.14.0
-modules-runtime@0.11.0
-mongo@1.7.0
+modules@0.15.0
+modules-runtime@0.12.0
+mongo@1.8.0
 mongo-decimal@0.1.1
 mongo-dev-server@1.1.0
 mongo-id@1.0.7
-npm-mongo@3.2.0
+npm-mongo@3.3.0
 ordered-dict@1.1.0
-ostrio:spiderable-middleware@1.4.5
+ostrio:spiderable-middleware@1.5.0
 promise@0.11.2
 random@1.1.0
 reload@1.3.0
@@ -46,5 +46,5 @@ tinytest@1.1.0
 tracker@1.2.0
 underscore@1.0.10
 url@1.2.0
-webapp@1.7.5
+webapp@1.8.0
 webapp-hashing@1.0.9

lib/index.js

@@ -19,7 +19,7 @@ var _debug = function debug (...args) {
 };
 
 request.defaultOptions.debug       = false;
-request.defaultOptions.headers     = { 'User-Agent': 'spiderable-middleware/1.4.5', Accept: '*/*' };
+request.defaultOptions.headers     = { 'User-Agent': 'spiderable-middleware/1.5.0', Accept: '*/*' };
 request.defaultOptions.noStorage   = true;
 request.defaultOptions.rawBody     = true;
 request.defaultOptions.retry       = true;
@@ -182,9 +182,14 @@ module.exports = (function () {
 
       reqUrl += '/' + urlObj.pathname;
       reqUrl  = reqUrl.replace(/([^:]\/)\/+/g, '$1');
+      reqUrl  = (this.serviceURL + '/?url=' + encodeURIComponent(reqUrl));
+
+      if (req.headers['user-agent']) {
+        reqUrl += '&bot=' + encodeURIComponent(req.headers['user-agent']);
+      }
 
       var opts  = Object.assign({}, this.requestOptions, {
-        uri: (this.serviceURL + '/?url=' + encodeURIComponent(reqUrl)).replace(/([^:]\/)\/+/g, '$1'),
+        uri: reqUrl,
         auth: this.auth || false
       });
 
@@ -249,11 +254,11 @@ module.exports = (function () {
           next();
         });
 
-        res.on('close', function () {
-          _debug('[RES] ["close" event]:', arguments);
-          serviceReq.abort();
-          next();
-        });
+        // res.on('close', function () {
+        //   _debug('[RES] ["close" event]:', arguments);
+        //   serviceReq.abort();
+        //   next();
+        // });
 
         res.sendDate = false;
         serviceReq.send();