Added MongoDB support, drones and associated states saved to DB and r…

…estarted automatically along with Ishiki, drones stdout and stderr logged to DB against app/user

config.sample.json

@@ -7,6 +7,11 @@
     "min": "9080",
     "max": "10080"
   },
+  "mongodb": {
+    "host": "127.0.0.1",
+    "port": "27017",
+    "database": "ishiki"
+  },
   "haibu": {
     "env": "development",
     "advanced-replies": true,

index.js

@@ -22,6 +22,11 @@ app.config.defaults({
     min: 9080,
     max: 10080
   },
+  mongodb: {
+    host: '127.0.0.1',
+    port: 27017,
+    database: 'ishiki'
+  },
   haibu: {
     env: 'development',
     'advanced-replies': true,
@@ -36,6 +41,26 @@ app.config.defaults({
   }
 });
 
+//instantiate db
+var mongo = require('mongodb'),
+  Server = mongo.Server,
+  Db = mongo.Db,
+  Bson = mongo.BSONPure;
+
+var server = new Server(app.config.get('mongodb:host'), app.config.get('mongodb:port'), {auto_reconnect: true});
+
+db = new Db(app.config.get('mongodb:database'), server, {safe: true}, {strict: false});
+
+db.open(function(err, db) {
+  var mongo_path = app.config.get('mongodb:host') + ':' + app.config.get('mongodb:port') + '/' + app.config.get('mongodb:database');
+
+  if (!err) {
+    console.log('Connected to ' + mongo_path);
+  }else{
+    console.log('Could not connect to ' + mongo_path);
+  }
+});
+
 //extend drone
 drone.deployOnly = require('./lib/drone.extend');
 

lib/ishiki.js

@@ -1,7 +1,40 @@
 var crypto = require('crypto'),
-  nodev = require('./nodev');
+  nodev = require('./nodev'),
+  droneModel = require('../models/drone'),
+  logModel = require('../models/log');
 
 module.exports = function(app, haibu, path, fs, drone, proxy) {
+  //listen to drones
+
+  //user/app logs
+  function logInfo(type, msg, meta) {
+    if (msg.trim() != '' && meta && meta.app && meta.user) {
+      var data = {
+        type: type,
+        msg: msg,
+        user: meta.user,
+        app: meta.app,
+        ts: new Date()
+      };
+
+      logModel.add(data, function(){});
+    }
+  }
+
+  //saves state of drone
+  function switchState(started, pkg) {
+    pkg.started = started;
+
+    logInfo('info', 'Application ' + (started ? 'started' : 'stopped'), {name: pkg.app, user: pkg.user});
+    droneModel.createOrUpdate(pkg, function(){});
+  }
+
+  haibu.on('drone:stdout', logInfo);
+  haibu.on('drone:stderr', logInfo);
+  haibu.on('drone:start', function(type, msg){ switchState(true, msg.pkg) });
+  haibu.on('drone:stop', function(type, msg){ switchState(false, msg.pkg) });
+
+  //returns available port within provided range
   var port = app.config.get('port-range:min');
 
   function getPort ()
@@ -12,6 +45,57 @@ module.exports = function(app, haibu, path, fs, drone, proxy) {
       return false;
   }
 
+  function startDrone(pkg, userid, appid, callback) {
+    var drone_port = getPort();
+
+    if (drone_port) {
+      pkg.env = pkg.env || {};
+      pkg.env['PORT'] = drone_port;
+
+      //ensure package user and name match internally
+      pkg.user = userid;
+      pkg.name = appid;
+
+      drone.start(pkg, function(err, result) {
+        if (err)
+          callback(err);
+
+        //clear old routes
+        proxy.deleteBy({user: userid, name: appid});
+
+        pkg.host = result.host;
+        pkg.port = result.port;
+
+        //async update package in db
+        droneModel.createOrUpdate(pkg, function(){});
+
+        //load new routes
+        proxy.load(app.config.get('public-port'), pkg);
+
+        callback(null, {drone: result});
+      });
+    }else{
+      callback({message: 'No more ports available'});
+    }
+  }
+
+  //automatically start drones
+  droneModel.get({started: true}, function(err, results) {
+    if (err)
+      return;
+
+    results.forEach(function(result) {
+      result = droneModel.process(result, false);
+
+      startDrone(result, result.user, result.name, function(err, drone) {
+        if (err)
+          return console.log('Error starting ' + result.user + '/' + result.name + ' ' + (err.message || ''));
+
+        console.log('Started ' + result.user + '/' + result.name);
+      })
+    });
+  });
+
   app.router.path('/drones', function() {
     //list all drones
     this.get(function() {
@@ -70,33 +154,6 @@ module.exports = function(app, haibu, path, fs, drone, proxy) {
       var res = this.res,
         req = this.req;
 
-      function startDrone(pkg) {
-        var drone_port = getPort();
-
-        if (drone_port) {
-          pkg.env = pkg.env || {};
-          pkg.env['PORT'] = drone_port;
-
-          drone.start(pkg, function(err, result) {
-            if (err)
-              return haibu.sendResponse(res, 500, err);
-
-            //clear old routes
-            proxy.deleteBy({user: userid, name: appid});
-
-            pkg.host = result.host;
-            pkg.port = result.port;
-
-            //load new routes
-            proxy.load(app.config.get('public-port'), pkg);
-
-            haibu.sendResponse(res, 200, { drone: result });
-          });
-        }else{
-          haibu.sendResponse(res, 500, { message: 'No more ports available' });
-        }
-      }
-
       //clean up app
       drone.clean({user: userid, name: appid}, function() {
         //deploy
@@ -115,7 +172,12 @@ module.exports = function(app, haibu, path, fs, drone, proxy) {
                 if (err)
                   return haibu.sendResponse(res, 500, {message: err.message});
 
-                startDrone(pkg);
+                startDrone(pkg, userid, appid, function(err, result) {
+                  if (err)
+                    return haibu.sendResponse(res, 500, err);
+
+                  haibu.sendResponse(res, 200, result);
+                });
               });
             }
           }

lib/proxy.js

@@ -195,8 +195,13 @@ Proxy.prototype.getBy = function(port, match) {
         }
       }
 
-      if (matched)
+      if (matched) {
         routes[domain] = self.proxies[port].routes[domain];
+
+        //json-unfriendly 'target' gets added by http-proxy
+        if (routes[domain].target)
+          delete routes[domain].target;
+      }
     });
 
     return routes;

models/_base.js

@@ -0,0 +1,87 @@
+var Bson = require('mongodb').BSONPure;
+
+var BaseModel = exports.BaseModel = function(options) {
+  options = options || {};
+
+  this.collection = options.collection || '';
+};
+
+//retrieve one entry if id passed, or several if {} passed
+BaseModel.prototype.get = function(filter, callback) {
+  if (!callback) {
+    callback = filter;
+    filter = {};
+  }
+
+  db.collection(this.collection, function(err, collection) {
+    if (err) {
+      callback(err);
+    }else{
+      if (typeof filter == 'string')
+      {
+        collection.findOne({_id: new Bson.ObjectID(filter)}, function(err, data) {
+          if (err)
+            callback(err);
+          else
+            callback(null, data);
+        });
+      }
+      else if (typeof filter == 'object') {
+        collection.find(filter).toArray(function(err, data) {
+          if (err)
+            callback(err);
+          else
+            callback(null, data);
+        });
+      }
+    }
+  });
+};
+
+//add entry
+BaseModel.prototype.add = function(data, callback) {
+  db.collection(this.collection, function(err, collection) {
+    if (err) {
+      callback(err);
+    }else{
+      collection.insert(data, function(err, result) {
+        if (err)
+          callback(err);
+        else
+          callback(null, result);
+      })
+    }
+  });
+};
+
+//update entry
+BaseModel.prototype.edit = function(id, data, callback) {
+  db.collection(this.collection, function(err, collection) {
+    if (err) {
+      callback(err);
+    }else{
+      collection.update({_id: new Bson.ObjectID(id)}, data, {safe: true}, function(err, result) {
+        if (err)
+          callback(err);
+        else
+          callback(null, result);
+      });
+    }
+  })
+};
+
+//delete entry
+BaseModel.prototype.delete = function(id, callback) {
+  db.collection(this.collection, function(err, collection) {
+    if (err) {
+      callback(err);
+    }else{
+      collection.remove({_id: Bson.ObjectID(id)}, {safe: true}, function(err, result) {
+        if (err)
+          callback(err);
+        else
+          callback(null, result);
+      });
+    }
+  });
+};

models/drone.js

@@ -0,0 +1,67 @@
+var BaseModel = require('./_base').BaseModel,
+  util = require('util');
+
+//mongodb illegal key chars
+var illegal_chars = ['/', '\\', '.', ' ', '"', '*', '<', '>', ':', '|', '?'],
+  escape_with = '__';
+
+//processes mongodb illegal characters in keys
+//pre to true preprocesses, otherwise postprocesses
+BaseModel.prototype.process = function(data, pre) {
+  var self = this;
+
+  Object.keys(data).forEach(function(key) {
+    var new_key = key;
+
+    illegal_chars.forEach(function(c, i) {
+      var from = pre ? c : escape_with + i + escape_with,
+        to = pre ? escape_with + i + escape_with : c;
+
+      if (key.indexOf(from) !== -1) {
+        new_key = new_key.split(from).join(to);
+      }
+    });
+
+    if (new_key != key) {
+      data[new_key] = data[key];
+      delete data[key];
+      key = new_key;
+    }
+
+    if (data[key] && typeof data[key] == 'object')
+      data[key] = self.process(data[key], pre);
+  });
+
+  return data;
+};
+
+BaseModel.prototype.createOrUpdate = function(pkg, callback) {
+  var self = this;
+
+  if (pkg._id) delete pkg._id;
+
+  this.get({name: pkg.name, user: pkg.user}, function(err, result) {
+    if (err)
+      return callback(err);
+
+    pkg = self.process(pkg, true);
+
+    if (result.length == 1) {
+      self.edit(result[0]._id.toString(), {$set: pkg}, function(err, success) {
+        if (err)
+          return callback(err);
+
+        callback(null, self.process(pkg, false));
+      });
+    }else{
+      self.add(pkg, function(err, result) {
+        if (err)
+          return callback(err);
+
+        callback(null, self.process(result[0], false));
+      });
+    }
+  });
+};
+
+module.exports = new BaseModel({collection: 'drone'});

models/log.js

@@ -0,0 +1,3 @@
+var BaseModel = require('./_base').BaseModel;
+
+module.exports = new BaseModel({collection: 'log'});

package.json

@@ -15,15 +15,17 @@
     "ikishi",
     "carapace",
     "http-proxy",
-    "automated node versions"
+    "automated node versions",
+    "deployment"
   ],
   "dependencies": {
     "union": "0.3.6",
     "flatiron": "0.3.3",
     "haibu": "0.9.7",
     "semver": "1.1.2",
     "tar": "0.1.14",
-    "http-proxy": "0.8.7"
+    "http-proxy": "0.8.7",
+    "mongodb": "1.2.x"
   },
   "main": "index.js",
   "preferGlobal": true,