Add proper configuration object checks

This change adds a bunch of checks to make sure the configuration file
does not makes the process crash if its missing some optional sections,
like formatters, blacklist, types, default values and http auth.

It also makes setting index setting optional.

Finally it makes sure that loading and invalid configuration file will
print a (more than currently) friendly message.

Fixes #19
Fixes #20

app.js

@@ -9,29 +9,66 @@
 var argv = require('optimist').argv;
 var pack = require('./package.json');
 var path = require('path');
-var config = require(argv.config ? path.resolve(argv.config) : './config.json');
 var fs = require('fs');
 var u = require('url');
 var _ = require('lodash');
+var algoliasearch = require('algoliasearch');
 
 var processOne = require('./lib/process');
 var sitemap = require('./lib/sitemap');
 var dns = require('./lib/dns-cache');
+var sitemapProcessed = 0;
 var sitemapCount = 0;
 var urlCount = 0;
 
-var algoliasearch = require('algoliasearch');
+var configFile = argv.config ? path.resolve(argv.config) : './config.json';
+var config = {};
+try {
+	config = require(configFile);
+} catch (ex) {
+	config = null;
+}
+
+if (!_.isObject(config)) {
+	console.error('Invalid configuration');
+	process.exit(1);
+}
+if (!_.isObject(config.cred)) {
+	console.error('Invalid credentials');
+	process.exit(2);
+}
+if (!_.isObject(config.index)) {
+	console.error('Invalid index configuration');
+	process.exit(4);
+}
+if (!_.isArray(config.sitemaps)) {
+	console.error('Invalid sitemaps configuration');
+	process.exit(8);
+}
+
 var client = algoliasearch(config.cred.appid, config.cred.apikey);
 var pages = client.initIndex(config.index.name);
 
 // Welcome
 console.log('Welcome to "%s" %s v%s', config.app, pack.name, pack.version);
 console.log();
+console.log('Loaded "%s" configuration', configFile);
+console.log();
 
 // Launch sitemap crawling
 sitemap(config, function (sitemap, urls) {
+	sitemapProcessed++;
+
 	if (!urls.length) {
 		console.log('Sitemap %s do not contains any urls', sitemap.url);
+	}
+
+	// All sitemaps have failed
+	if (sitemapProcessed === config.sitemaps.length && sitemapCount === 0) {
+		console.log();
+		process.exit(-1);
+	} else if (!urls.length) {
+		// Current sitemap failed, exit
 		return;
 	}
 
@@ -94,37 +131,40 @@ sitemap(config, function (sitemap, urls) {
 });
 
 // Configure index
-console.log('Configuring your index %s', config.index.name);
-pages.setSettings(config.index.settings, function (error, result) {
-	if (!!error) {
-		console.log();
-		if (!!result && !!result.message) {
-			console.error('Error! ' + result.message);
-		}
-		if (!!error && !!error.message) {
-			console.error('Error! ' + error.message);
+if (_.isObject(config.index.settings)) {
+	console.log('Configuring your index %s', config.index.name);
+	pages.setSettings(config.index.settings, function (error, result) {
+		if (!!error) {
+			console.log();
+			if (!!result && !!result.message) {
+				console.error('Error! ' + result.message);
+			}
+			if (!!error && !!error.message) {
+				console.error('Error! ' + error.message);
+			}
+		} else {
+			console.log('Configured index properly');
+			console.log();
 		}
-		console.log();
-	} else {
-		console.log('Configured index properly');
-		console.log();
-	}
-});
+	});
+}
 
 var removeOldEntries = function () {
 	urlCount++;
-	if (urlCount === sitemapCount && !!config.oldentries) {
+	if (urlCount === sitemapCount) {
 		processOne.stop();
-		console.log()
-		console.log('Removing old entries...');
-		pages.deleteByQuery('*', {
-			numericFilters: ['timestamp<' + (new Date().getTime() - config.oldentries)]
-		}, function (error, content) {
-			if (!!error) {
-				console.error('Error deleting entries.');
-				return;
-			}
-			console.log('Deleting old entries done.');
-		});
+		if (_.isInteger(config.oldentries) && config.oldentries > 0) {
+			console.log()
+			console.log('Removing old entries...');
+			pages.deleteByQuery('*', {
+				numericFilters: ['timestamp<' + (new Date().getTime() - config.oldentries)]
+			}, function (error, content) {
+				if (!!error) {
+					console.error('Error deleting entries.');
+					return;
+				}
+				console.log('Deleting old entries done.');
+			});
+		}
 	}
 };

lib/process.js

@@ -84,7 +84,7 @@ var parse = function (record, data, config) {
 			});
 
 			// A formatter for this key does exists
-			if (!!config.formatters[key]) {
+			if (!!config.formatters && !!config.formatters[key]) {
 				if (!_.isArray(config.formatters[key])) {
 					config.formatters[key] = [config.formatters[key]];
 				}
@@ -99,7 +99,8 @@ var parse = function (record, data, config) {
 				});
 			}
 
-			if (!!config.types[key]) {
+			// A type converter for this key does exists
+			if (!!config.types && !!config.types[key]) {
 				// Cast all values
 				record[key] = _.map(record[key], function (value) {
 					if (!!types[config.types[key]]) {
@@ -108,7 +109,9 @@ var parse = function (record, data, config) {
 				});
 			}
 
-			if (!record[key].length && !!config.defaults[key]) {
+			// The key currently has a falsy value and
+			// a default value for this key does exists
+			if (!record[key] && !!config.defaults && !!config.defaults[key]) {
 				record[key] = config.defaults[key];
 			}
 		} else {
@@ -136,7 +139,7 @@ module.exports = function (config, url, cb) {
 		port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80),
 		path: parsedUrl.pathname || '/',
 		method: 'GET',
-		auth: config.http.auth
+		auth: config.http && config.http.auth
 	};
 
 	if (!httpOptions.hostname) {

lib/sitemap.js

@@ -19,7 +19,7 @@ module.exports = function (config, cb) {
 			port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80),
 			path: parsedUrl.path,
 			method: 'GET',
-			auth: config.http.auth
+			auth: config.http && config.http.auth
 		};
 		var client = parsedUrl.protocol === 'https:' ? https : http;