Add maxRecordSize (#31)

* Add record trimmer d102dbb
* rename trimsize to maxRecordSize  378d102
* add trimsize description b2f1002

Fixes #6

README.md

@@ -95,6 +95,10 @@ The maximum number of seconds an entry can live without being updated. After
 each run, the app will search for old entries and delete them. If you do not
 wish to get rid of old entries, set this value to 0.
 
+#### maxRecordSize: Integer
+
+The maximum size in bytes of a record to be sent to Algolia. The default is 10,000 but could vary based on different plans.
+
 #### index: Object
 
 An object containing various values related to your index.

config.json

@@ -5,6 +5,7 @@
 		"apikey": "<ALGOLIA API KEY>"
 	},
 	"oldentries" : 86400000,
+	"maxRecordSize": 10000,
 	"index": {
 		"name": "<my index>",
 		"settings": {

lib/process.js

@@ -139,7 +139,15 @@ const parse = (record, data, config) => {
 		}
 	});
 };
-
+const trimmer = (record, config) => {
+	const bytes = (s) => { return ~-encodeURI(s).split(/%..|./).length }
+	const jsonSize = (s) => { return bytes(JSON.stringify(s)) }
+	const limit = config.maxRecordSize;
+
+	while (jsonSize(record) > limit && record.text.length > 0) {
+		record.text.pop();
+	}
+}
 module.exports = (data, cb) => {
 	const config = data.config;
 	const plugins = data.plugins;
@@ -217,6 +225,7 @@ module.exports = (data, cb) => {
 				let error = null;
 				try {
 					parse(record, data, config);
+					trimmer(record, config);
 					plugins(record, data);
 				} catch (ex) {
 					error = ex;