Merge branch 'unstable' into UI

README.md

@@ -1,4 +1,4 @@
-![zenbot logo](https://rawgit.com/carlos8f/zenbot/master/assets/logo.png)
+![zenbot logo](https://rawgit.com/deviavir/zenbot/master/assets/logo.png)
 
 > “To follow the path, look to the master, follow the master, walk with the master, see through the master, become the master.”
 > – Zen Proverb
@@ -50,13 +50,13 @@ Zenbot is a command-line cryptocurrency trading bot using Node.js and MongoDB. I
 Run in your console,
 
 ```
-git clone https://github.com/carlos8f/zenbot.git
+git clone https://github.com/deviavir/zenbot.git
 ```
 
 Or, without git,
 
 ```
-wget https://github.com/carlos8f/zenbot/archive/master.tar.gz
+wget https://github.com/deviavir/zenbot/archive/master.tar.gz
 tar -xf zenbot-master.tar.gz
 mv zenbot-master zenbot
 ```
@@ -93,7 +93,7 @@ sudo apt-get install build-essential mongodb -y
 curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
 sudo apt-get install -y nodejs
 
-git clone https://github.com/carlos8f/zenbot.git
+git clone https://github.com/deviavir/zenbot.git
 cd zenbot
 npm install
 
@@ -476,7 +476,7 @@ In it's infancy, there are a few caveats with the current UI.
 
 ## Reading the console output
 
-![console](https://rawgit.com/carlos8f/zenbot/master/assets/console.png)
+![console](https://rawgit.com/deviavir/zenbot/master/assets/console.png)
 
 From left to right:
 
@@ -674,7 +674,7 @@ zenbot sell gdax.BTC-USD --pct=10
 
 ## Chat with other Zenbot users
 
-[![zenbot logo](https://rawgit.com/carlos8f/zenbot/master/assets/discord.png)](https://discord.gg/ZdAd2gP)
+[![zenbot logo](https://rawgit.com/deviavir/zenbot/master/assets/discord.png)](https://discord.gg/ZdAd2gP)
 
 Zenbot has a Discord chat! You can get in [through this invite link](https://discord.gg/ZdAd2gP).
 
@@ -686,7 +686,7 @@ P.S., some have asked for how to donate to Zenbot development. I accept donation
 
 `187rmNSkSvehgcKpBunre6a5wA5hQQop6W`
 
-![zenbot logo](https://rawgit.com/carlos8f/zenbot/master/assets/zenbot_square.png)
+![zenbot logo](https://rawgit.com/deviavir/zenbot/master/assets/zenbot_square.png)
 
 Thanks!
 

commands/backfill.js

@@ -5,6 +5,9 @@ var tb = require('timebucket')
 
 module.exports = function container (get, set, clear) {
   var c = get('conf') || {}
+
+  var collectionService = get('lib.collection-service')(get, set, clear)
+
   return function (program) {
     program
       .command('backfill [selector]')
@@ -17,10 +20,10 @@ module.exports = function container (get, set, clear) {
           console.error('cannot backfill ' + selector.normalized + ': exchange not implemented')
           process.exit(1)
         }
-        var trades = get('db.trades')
-        get('db.mongo').collection('trades').ensureIndex({selector: 1, time: 1})
-        var resume_markers = get('db.resume_markers')
-        get('db.mongo').collection('resume_markers').ensureIndex({selector: 1, to: -1})
+
+        var trades = collectionService.getTrades();
+        var resume_markers = collectionService.getResumeMarkers();
+
         var marker = {
           id: crypto.randomBytes(4).toString('hex'),
           selector: selector.normalized,

extensions/exchanges/binance/exchange.js

@@ -220,7 +220,7 @@ module.exports = function container (get, set, clear) {
         cb(null, order)
       }).catch(function (error) {
         console.error('An error occurred', error)
-        return retry('buy', func_args)
+        return retry('sell', func_args)
       })
     },
 

extensions/exchanges/gdax/exchange.js

@@ -1,20 +1,53 @@
 var Gdax = require('gdax')
-  , path = require('path')
-  , colors = require('colors')
-  , numbro = require('numbro')
 
 module.exports = function container (get, set, clear) {
   var c = get('conf')
 
-  var public_client = {}, authed_client
+  var public_client = {}, authed_client, websocket_client = {}, websocket_cache = {}
 
   function publicClient (product_id) {
     if (!public_client[product_id]) {
+      websocketClient(product_id)
       public_client[product_id] = new Gdax.PublicClient(product_id, c.gdax.apiURI)
     }
     return public_client[product_id]
   }
 
+  function websocketClient (product_id) {
+    if (!websocket_client[product_id]) {
+      // OrderbookSync extends WebsocketClient and subscribes to the 'full' channel, so we can use it like one
+      var auth = null
+      try {
+        auth = authedClient()
+      } catch(e){}
+      websocket_client[product_id] = new Gdax.OrderbookSync([product_id], c.gdax.apiURI, c.gdax.websocketURI, auth)
+      // initialize a cache for the websocket connection
+      websocket_cache[product_id] = {
+        trades: [],
+        trade_ids: []
+      }
+      websocket_client[product_id].on('open', () => {
+        console.log('websocket connection to '+product_id+' opened')
+      })
+      websocket_client[product_id].on('message', (message) => {
+        switch (message.type) {
+        case 'match':
+          handleTrade(message, product_id)
+          break
+        default:
+          break
+        }
+      })
+      websocket_client[product_id].on('error', (err) => {
+        console.log(err)
+      })
+      websocket_client[product_id].on('close', () => {
+        console.error('websocket connection to '+product_id+' closed, attempting reconnect')
+        websocket_client[product_id].connect()
+      })
+    }
+  }
+
   function authedClient () {
     if (!authed_client) {
       if (!c.gdax || !c.gdax.key || c.gdax.key === 'YOUR-API-KEY') {
@@ -45,6 +78,15 @@ module.exports = function container (get, set, clear) {
     }, 10000)
   }
 
+  function handleTrade(trade, product_id) {
+    var cache = websocket_cache[product_id]
+    cache.trades.push(trade)
+    cache.trade_ids.push(trade.trade_id)
+  }
+
+  // TODO: this contains open orders and gets updated on buy/sell/getOrder
+  // should maintain a list of ID's and keep this up to date from the websocket feed's 
+  // 'done'/'change'/'match' messages and use this as a cache for `getOrder` below
   var orders = {}
 
   var exchange = {
@@ -69,6 +111,30 @@ module.exports = function container (get, set, clear) {
         // move cursor into the past
         args.after = opts.to
       }
+      // check for locally cached trades from the websocket feed
+      var cache = websocket_cache[opts.product_id]
+      var max_trade_id = cache.trade_ids.reduce(function(a, b) {
+        return Math.max(a, b)
+      }, -1)
+      if (opts.from && max_trade_id >= opts.from) {
+        var fromIndex = cache.trades.findIndex((value)=> {return value.trade_id == opts.from})
+        var newTrades = cache.trades.slice(fromIndex + 1)
+        newTrades = newTrades.map(function (trade) {
+          return {
+            trade_id: trade.trade_id,
+            time: new Date(trade.time).getTime(),
+            size: Number(trade.size),
+            price: Number(trade.price),
+            side: trade.side
+          }
+        })
+        newTrades.reverse()
+        cb(null, newTrades)
+        // trim cache
+        cache.trades = cache.trades.slice(fromIndex)
+        cache.trade_ids = cache.trade_ids.slice(fromIndex)
+        return
+      }
       client.getProductTrades(args, function (err, resp, body) {
         if (!err) err = statusErr(resp, body)
         if (err) return retry('getTrades', func_args, err)
@@ -108,6 +174,22 @@ module.exports = function container (get, set, clear) {
     },
 
     getQuote: function (opts, cb) {
+      // check websocket cache first
+      if(websocket_client[opts.product_id] && websocket_client[opts.product_id].books) {
+        var book = websocket_client[opts.product_id].books[opts.product_id]
+        var state = book.state()
+        var asks = state.asks
+        var bids = state.bids
+        if(bids.length && asks.length){
+          // price is a `num` arbitrary precision number and needs to be toString()ed and parseFloat()ed 
+          var best_bid = parseFloat(bids[0].price.toString())
+          var best_ask = parseFloat(asks[0].price.toString())
+          if(best_bid && best_ask){
+            cb(null, {bid: best_bid, ask: best_ask})
+            return
+          }
+        }
+      }
       var func_args = [].slice.call(arguments)
       var client = publicClient(opts.product_id)
       client.getProductTicker(function (err, resp, body) {

extensions/exchanges/kraken/exchange.js

@@ -48,7 +48,7 @@ module.exports = function container(get, set, clear) {
     if (assetsToFix.indexOf(asset) >= 0 && currency.length > 3) {
       currency = currency.substring(1)
     }
-    return `X${asset}X${currency}`
+    return asset + currency;
   }
 
   function retry(method, args, error) {

lib/_codemap.js

@@ -21,5 +21,6 @@ module.exports = {
   'adx': require('./adx'),
   'vwap': require('./vwap'),
   'slow_stochastic': require('./slow_stochastic'),
+  'collection-service': require('./services/collection-service'),
   'cmf': require('./cmf')
 }

lib/services/collection-service.js

@@ -0,0 +1,21 @@
+module.exports = (function (get, set, clear) {
+
+	return {
+
+		getTrades: () => {
+	        var trades = get('db.trades')
+	        get('db.mongo').collection('trades').ensureIndex({selector: 1, time: 1})
+
+	        return trades
+		},	
+
+		getResumeMarkers: () => {
+	        var resume_markers = get('db.resume_markers')
+	        get('db.mongo').collection('resume_markers').ensureIndex({selector: 1, to: -1})
+
+	        return resume_markers;
+		}
+	}
+
+})
+