diff --git a/app.js b/app.js index 1000321..2be4916 100644 --- a/app.js +++ b/app.js @@ -90,41 +90,43 @@ sitemap(config, function (sitemap, urls) { var results = _.map(urls, function (url, index) { console.log('Registered ' + url.url); - var processResults = processOne(config, url, function (error, record) { - if (!!error || !record) { - console.error('Error! ' + error.message); - if (!!error.pageNotFound && !!record) { - pages.deleteObject(record.objectID, function (error, result) { - console.log('Object ' + record.objectID + ' has been deleted'); - }); - } - removeOldEntries(); - return; - } - - pages.saveObject(record, function (error, result) { - if (!!error) { - console.log(); - if (!!result && !!result.message) { - console.error('Error! ' + result.message); - } - if (!!error && !!error.message) { - console.error('Error! ' + error.message); + setTimeout(function () { + var processResults = processOne(config, url, function (error, record) { + if (!!error || !record) { + console.error('Error! ' + error.message); + if (!!error.pageNotFound && !!record) { + pages.deleteObject(record.objectID, function (error, result) { + console.log('Object ' + record.objectID + ' has been deleted'); + }); } - console.log(); - } else if (record.objectID !== result.objectID) { - console.log(); - console.error('Error! Object ID mismatch!'); - console.log(); - } else { - console.log('Object %s:%s saved (%s)', record.objectID, record.lang, record.url); + removeOldEntries(); + return; } - removeOldEntries(); + + pages.saveObject(record, function (error, result) { + if (!!error) { + console.log(); + if (!!result && !!result.message) { + console.error('Error! ' + result.message); + } + if (!!error && !!error.message) { + console.error('Error! ' + error.message); + } + console.log(); + } else if (record.objectID !== result.objectID) { + console.log(); + console.error('Error! Object ID mismatch!'); + console.log(); + } else { + console.log('Object %s:%s saved (%s)', record.objectID, record.lang, record.url); + } + removeOldEntries(); + }); }); - }); - if (!processResults.ok) { - console.error(processResults.message || 'Error!'); - } + if (!processResults.ok) { + console.error(processResults.message || 'Error!'); + } + }, config.delayBetweenRequest * index); }); console.log('Sitemap %s registered %s / %s urls', sitemap.url, results.length, urls.length); diff --git a/config.json b/config.json index c4dc170..99aa148 100644 --- a/config.json +++ b/config.json @@ -17,8 +17,12 @@ {"url": "http://example.com/en/sitemap.xml", "lang": "en"} ], "http": { - "auth": "" + "auth": "", + "headers": { + + } }, + "delayBetweenRequest": 100, "selectors": { "title": "title", "image": "meta[property=\"og:image\"]", diff --git a/lib/process.js b/lib/process.js index 19501e6..d64609f 100644 --- a/lib/process.js +++ b/lib/process.js @@ -139,7 +139,8 @@ module.exports = function (config, url, cb) { port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80), path: parsedUrl.pathname || '/', method: 'GET', - auth: config.http && config.http.auth + auth: config.http && config.http.auth, + headers: config.http.headers }; if (!httpOptions.hostname) { diff --git a/lib/sitemap.js b/lib/sitemap.js index 4e1e3c1..ff7bfbc 100644 --- a/lib/sitemap.js +++ b/lib/sitemap.js @@ -19,7 +19,8 @@ module.exports = function (config, cb) { port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80), path: parsedUrl.path, method: 'GET', - auth: config.http && config.http.auth + auth: config.http && config.http.auth, + headers: config.http.headers }; var client = parsedUrl.protocol === 'https:' ? https : http;