From 692cbe8b79dfb3821af3946f787d87e5ff9bd4ee Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Sun, 26 Jun 2016 21:37:23 -0700 Subject: [PATCH 0001/1659] init gdax recorder --- _codemap.js | 4 +--- conf/_codemap.js | 20 ++++--------------- controllers/_codemap.js | 9 --------- controllers/home.js | 6 ------ db/_codemap.js | 4 ++-- db/{users.js => messages.js} | 2 +- hooks/_codemap.js | 27 +------------------------ hooks/mountRecorder.js | 37 +++++++++++++++++++++++++++++++++++ middleware/_codemap.js | 13 ------------ middleware/vars.js | 13 ------------ package.json | 5 ++++- public/404.html | 12 ------------ public/motley-full.png | Bin 11746 -> 0 bytes public/style.css | 26 ------------------------ server.js => recorder.js | 14 +++++++++---- utils/_codemap.js | 5 +++++ utils/gdaxWebsocket.js | 5 +++++ vendor/_codemap.js | 8 -------- views/index.hbs | 4 ---- views/layout.hbs | 12 ------------ 20 files changed, 70 insertions(+), 156 deletions(-) delete mode 100644 controllers/_codemap.js delete mode 100644 controllers/home.js rename db/{users.js => messages.js} (90%) create mode 100644 hooks/mountRecorder.js delete mode 100644 middleware/_codemap.js delete mode 100644 middleware/vars.js delete mode 100644 public/404.html delete mode 100644 public/motley-full.png delete mode 100644 public/style.css rename server.js => recorder.js (66%) create mode 100644 utils/_codemap.js create mode 100644 utils/gdaxWebsocket.js delete mode 100644 vendor/_codemap.js delete mode 100644 views/index.hbs delete mode 100644 views/layout.hbs diff --git a/_codemap.js b/_codemap.js index 77331fdd1c..50352abd44 100644 --- a/_codemap.js +++ b/_codemap.js @@ -2,10 +2,8 @@ module.exports = { _ns: 'motley', _maps: [ require('./conf/_codemap'), - require('./controllers/_codemap'), require('./db/_codemap'), require('./hooks/_codemap'), - require('./middleware/_codemap'), - require('./vendor/_codemap') + require('./utils/_codemap') ] } \ No newline at end of file diff --git a/conf/_codemap.js b/conf/_codemap.js index 065c4565b8..fcd1f462a6 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -3,21 +3,9 @@ module.exports = { _ns: 'motley', _folder: 'conf', - // site overrides - '@site.port': 3000, - '@site.title': 'your new Motley site', - - // middleware overrides - 'middleware.session{}': { - cookie: { - maxAge: 86400 * 365 // session cookie lifetime: 1 year - }, - key: 'motley' // change this to customize the session cookie name - }, - 'middleware.templ{}': { - watch: true // watch for template changes and auto-recompile - }, - 'middleware.buffet{}': { - watch: true // watch for file changes and auto-clear cache + '@db.mongo': { + url: 'mongodb://localhost:27017/gdax_history', + username: null, + password: null } } \ No newline at end of file diff --git a/controllers/_codemap.js b/controllers/_codemap.js deleted file mode 100644 index 59c514d1b2..0000000000 --- a/controllers/_codemap.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = { - // meta - _ns: 'motley', - - 'controllers[]': [ - require('./home') - // add more controllers here. - ] -} \ No newline at end of file diff --git a/controllers/home.js b/controllers/home.js deleted file mode 100644 index 975906207d..0000000000 --- a/controllers/home.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = function container (get, set) { - return get('controller')() - .get('/', function (req, res, next) { - res.render('index') - }) -} \ No newline at end of file diff --git a/db/_codemap.js b/db/_codemap.js index 831a42bc22..0c55b31bc9 100644 --- a/db/_codemap.js +++ b/db/_codemap.js @@ -4,11 +4,11 @@ module.exports = { _folder: 'db', // named collections - 'users': require('./users'), + 'messages': require('./messages'), // collection registration 'collections[]': [ - '#db.users' + '#db.messages' // add more collections here. ] } \ No newline at end of file diff --git a/db/users.js b/db/messages.js similarity index 90% rename from db/users.js rename to db/messages.js index d1eb8e90c7..1a91ad03d8 100644 --- a/db/users.js +++ b/db/messages.js @@ -1,5 +1,5 @@ module.exports = function container (get, set) { - return get('db.createCollection')('users', { + return get('db.createCollection')('messages', { load: function (obj, opts, cb) { // respond after the obj is loaded cb(null, obj); diff --git a/hooks/_codemap.js b/hooks/_codemap.js index 47f152d696..b7157a3f17 100644 --- a/hooks/_codemap.js +++ b/hooks/_codemap.js @@ -2,30 +2,5 @@ module.exports = { // meta _ns: 'motley', _folder: 'hooks', - - // core hook registration - 'boot[]': function container (get, set) { - return function task (cb) { - // respond to the boot hook - setImmediate(cb) - } - }, - 'mount[]': function container (get, set) { - return function task (cb) { - // respond to the mount hook - setImmediate(cb) - } - }, - 'listen[]': function container (get, set) { - return function task (cb) { - get('console').log('listening on http://localhost:' + get('site.server').address().port + '/') - setImmediate(cb) - } - }, - 'close[1]': function container (get, set) { - return function task (cb) { - get('console').log('\n\nmotley says goodbye :)\n') - setImmediate(cb) - } - } + 'mount[]': require('./mountRecorder') } \ No newline at end of file diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js new file mode 100644 index 0000000000..2b8f242ba5 --- /dev/null +++ b/hooks/mountRecorder.js @@ -0,0 +1,37 @@ +var numeral = require('numeral') + +module.exports = function container (get, set) { + return function mountRecorder (cb) { + var socket = get('utils.gdaxWebsocket') + var counter = 0 + setInterval(function () { + var totalSize = trades.reduce(function (prev, curr) { + return prev + curr.size + }, 0) + var totalPrice = trades.reduce(function (prev, curr) { + return prev + curr.price + }, 0) + var avg = trades.length ? numeral(totalPrice / trades.length).format('$0,0') : 0 + get('console').log('saved ' + counter + ' messages. trades: ' + trades.length + ' ' + avg + ' / ' + numeral(totalSize).format('0.000')) + trades = [] + counter = 0 + }, 10000) + var trades = [] + socket.on('message', function (message) { + message.id = String(message.sequence) + message.time_date = new Date(message.time) + get('db.messages').save(message, function (err, saved) { + if (err) get('console').error('message save err', err) + counter++ + if (saved.type === 'match' && saved.product_id === 'BTC-USD') { + trades.push({ + size: parseFloat(saved.size), + price: parseFloat(saved.price) + }) + } + }) + }) + get('console').log('mounted GDAX recorder.') + cb() + } +} \ No newline at end of file diff --git a/middleware/_codemap.js b/middleware/_codemap.js deleted file mode 100644 index 93a2d5a004..0000000000 --- a/middleware/_codemap.js +++ /dev/null @@ -1,13 +0,0 @@ -module.exports = { - // meta - _ns: 'motley', - - // named middleware - 'middleware.vars': require('./vars'), - - // middleware registration - 'middleware[]': [ - '#middleware.vars' - // add more middleware here. - ] -} \ No newline at end of file diff --git a/middleware/vars.js b/middleware/vars.js deleted file mode 100644 index 13b6a8ad5f..0000000000 --- a/middleware/vars.js +++ /dev/null @@ -1,13 +0,0 @@ -module.exports = function container (get, set) { - return function vars (req, res, next) { - // access req or res here, before routes are run. - res.vars.title = get('conf.site.title') - res.vars.post = req.body - res.vars.user = req.user - res.vars.nonce = Math.random() - res.vars.welcome = 'to ' + get('conf.site.title') + '!' - res.vars.version = require('../package.json').version - res.vars.core = require('motley/package.json').version - next() - } -} \ No newline at end of file diff --git a/package.json b/package.json index fdf949d035..6e7e2ce3c0 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,11 @@ "version": "0.0.0", "description": "", "dependencies": { + "coinbase-exchange": "^0.2.1", "motley": "^2.2.0", "motley-buffet": "^2.0.1", - "motley-templ": "^2.0.2" + "motley-mongo": "^2.0.5", + "motley-templ": "^2.0.2", + "numeral": "^1.5.3" } } diff --git a/public/404.html b/public/404.html deleted file mode 100644 index 2d7fc8fb71..0000000000 --- a/public/404.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - Page not found - - - - -

Page not found!

- - \ No newline at end of file diff --git a/public/motley-full.png b/public/motley-full.png deleted file mode 100644 index ff75d68426f2154a112e6f6a948cd6985ba01205..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11746 zcma)iWmFsSvvzPVUffD4ZpGb8ad$$I;zfhIYmuVCp+JG+?rz21-JReT-QXF#Me*uO$CqF9ip+E zv?So=?<>2tApX^YV*gnO^cpeu_YEUXm*)0rL~@c*kV0BP!o~f_I>>j;4ggRBWF*DZ z-4;(WT+=mr??bcrZdXt64rZcJwN;LT5x&CUCsJ4#%TrbMX~%($zaw&eQsZP#BFwGC zio&k^WSIPW9-Apl5>Ht)d9{AkbL{aM3^I~3+*|fIJ%!#iQaIlp3xRfrhh0})SKX1| z#jyV8_8Ct0wzNX=^u{dIKscZ7{NbKc;H{~|JC@Pi2vdFALIXv`0w!l&xrNk5&wsTe_Xp> z8IErf!OO#2-P%XNngO=*n&8D=VS@}+;hm})%Hq`%DYscS9skk&KR_bCsj^Y9P(e>c zsYdd>Hpf>}zQ$6a>QYRPbx|HXZvND4Q5q(&&JZ?QL^)fS`dArKiEFg!u38^J8J3ME zKKnWBKDG$meuKA^m-F|*x+qmDEFQM0pxUQ8{IUQ-thsS}Rs3r!h9Q)5+dn%amJ#^4 ztq^iHf?do6{EWY$UF*U=Ru#UM!d&7uc=NE)g zROQAcm@k0*5&e3U{eeuiB~443kmi2LzOfDZmgdH^^E#bs|pxeMiy2X`eXl1GT z;)h~c`b&k?WcpHMzSK(|p8@MY#>RejW0a2dqKiy;u|H(53vv;VDMWXyKS1PgNA#%^ zm_3TS^vykWSYHhLQV>RZgU#NaMr0!OCzV|!AOjM#CZsKG$H$fEi2+&& ztcMcrECCI(S59~_2}(N$Ba980^Y%YNkZ4q;u;44lgJ=7)87?7IQw?vD`)hVZ^DQaw zVA5sj6K1FB2#Hi5UB}f!1PcxK(Jl;LJ3)*<7e%5V#Wxh+Gg<3GiD#5g*8OecVAcwu zY>u);=|uEq-G!`AD}Gllg}+$!wh&rCET3-VP%9Q4;v!r>TqrCQjw+j68?FRl7Y+g! zwK_%ut9*)+&0$Y!k2Q{8ok9EN8^SCcqd%swJTXY_U48i z@^|%{QRpiMtLGwn$eF>PNo2Rr@yRm3*rw5*G5Y($#(ZsD&h?Y?^AW{Vnzt#s{y=BV z291E+X41r>r=W6#h`H13S&z&Z;JHVNy0;-<2$7Cd2YvwEAOBz?<0LXO*P6DuHSah5 zJj}la+lE29j^gT>S!JS=ZEI^#Y5A*YG)N6&2J61I^g~3jt1M1_U@LlKF0L8cs!Xq>&Ig& zmK34%TSTS6lmh5O1g?L0CQ=_Oji_tnk(D2-%$lk`k^OfbcEyk+u=X$bEt(c|!n94o z+Jf^#VWs7@fliLMaCQ%D*`$yMw2MNy7dX1xb6XZz1KXm2Ot;S@X0Q@WWlKo$7{yAs zV-mVm!Cd#z4X!a0qMT9A=kJvL9qRMd{EATzCB5uV7lZn&y*rq%heB z6JPaM;tHd6-_SN{%~*VC5q-M1;9-&^%^@p?9C7qUa0i3foBe3>Aibh6r)4nA-JM3nku%x-eoF6Zw=4(>v?M+GXk<^* z510P9_U$cIMLaY~o@Cv{PWk0r(Ks{QW9`0q37`ZAao>`!Yv4vHehw0JFPMY2_*ilG z2+jp$zCpVc{2polEeY{gk885=?2Mhlh35@ylEY<3In;eKL3T0gyru99S}r>HNXIM(;4aE(pUU&B z4-SS3&2G1Gr>{pviv6MxFi!t=2(A65v)B0c3Pa`@IG=VOW&19SA(U)eew}LQXbk3q z$+BVN={AZA1;%oG6p5lBW{6_G1hVaQU4E?2k*CC-CvUDpWpH?Ux#%E7v|fA1@w>tI?PvET5^SCqfe~vc(wYcD$0@J;odVD!s#~i}sO_N4eDiMFs%N~`X)pN1OpmRT z%V;=N-@N=02{!b-&U4STNZ+?e4~`X95Bo$u?{D__$s1J!H=)HA_@AUB?a}54Bt2Sc z?Vx4FXsG;l5utMR!Q#DK8xX7PETdB2?+BAVa6NKwRdpz2i zgZNns0(&lgm9toJzM|anY}p!ow5d*k+88 zv1>$&&)U!tG&GOyu-{{AHk{~pGSpv>&DJWdW7xa({5TiTwgqBmS6aN#A*UGGXTA>D+OXanqnp{TnT5gM6`gcFQm*ZCnE zo~f(3tBg=PKg>7_`+DP>8d`xNX3AE*3jkgDCY?UHrd5pRNWC8AxkV0cF(aDBm{9YU zPCWm0i`zgz*St&a+u=8CmqNr&n}SX|PiiaHawEliUN<`}R5K*R5yoXM!*%RwJVio! zU+C3y#PNA~k?_?fn`1ZQuyUf)9)`=b zctQbFGA#FzU85ug?0OlKQem@|HYX0lQdh^J3ZkvHtUPkk`kCUS37&7nRgZPrBeY z_?x!+i}lCV2O|58roA~fB=t_rq!p$_n9QC4f_#~D>DA*vi&Eu5F_kY^k>!haM|Fb` zuPU$Sb>}o$9!~@MT2J=!@U7@m`Q)6FgT+$6I2WpSRs{G&Q~M(h#9G|S&|&^IHygAx zSyFKy&zT#c!?VYdjmPJpxs9muB{>`RrH@ajWj7_|%}iz9`0+ZGbWs7qYiav+>SDey zysv{qI|}Y{GaZI0JuD)C2hbhA4^@I)8x|#t>TzSKjj3{?*7lr46H>4=zekVs>mG2t z>D5Pls0ANea|g<^j^S6+3d1ADugY7Hu`LO{xm+N;MG)@)R^U0T>qod_s|*P!O`un4sr#}kyTxE@WyoFMb~IXZ_g6&if8 z=ddN-B&X)}@;G}K8T(Rcv0(1aGso`$K}+*0EuIKXR&{Y!dUL2LsxoZcJ1LL_#HYI} zOd9childP`DvT`5(|`J*9rU9dUD1=feIs*biuC2Whn@}|IPSaS-t*{oo}jggawpfa z*}P=5yn=L}{_PR5fe&s+C-7W#(XrNCA3nIcWgzWDppEcLAzV;(U)Dy`b*~B0S7G<~W+o#Wm0CHHR~p z%MZ^5y-W6ZyU|=MrzcQ{KYptG9{t%?BBz%<*Sfhc%|BS54Mz5`cgC)ZxadM>I6@uD zkUiEBo{|Y!b19Uu8n0k2l>>T!)M%PlvxArX!aC1c!TeduN=V=mBwfHlx8VMRUl~B zI%$7(oVj944qEgx$f{$u{l5N|pBIy?=SP3igeCM^*ly|S0mMIk^sstm>EuuwGw+!{ zzL*cX$9p4<{830qZnjRuUx%6^@O;r`K$WkS=b*3x58T5GCB5-1wMmKA@t8>3;hiYJ zj+1$Fx!P#+rHw}mVI)7B=hwV);#`sKh^o7dT0X;A)!mXUvFJPO(ff`-9R?`(@PW}2nW*w1y!6&O^uM@vu7CHIq^0xysqUWO(#yU3-d zDw`Va%(c=Trzb%C^}gGN(W}j|RlG8YwI}b7*`tIjnSrPCS-PB;X4yJH zI!RT)0KH>{SlXS!77egTa{Vdiz&sfZmz}mrEAH`=G(M?^e^{{&ce! zyF(J z2J(jQqvK3988o*q+&80aB#S5f-FOp+b-Bl`XS_n1?1>?uh*XXJOAKsMFg4)nMQpPd zm%&*(!af^nZiKe$G^9zpk0GPG3&=9O@iI$8s&FMaDSMeycaGbOFUMH_9YuM zSB}^v#fw52YVk3d3(&I5^pSshSf$}*X~qGiv*pf>(IlwwAw@|H@O>$a-t%f)naG+& z38X#D6_~I#Bw2g_v(e=jQupPXyrqPXDW1=)`5Y%?bZWJX9inMDt_z{aO((cHI$_dIV`JW}ZD5*WH* z-0hCwiuLnpTfWN_J#Z5&S$&-9$=$@DF-j+w(X=KwpBJ|*wjAnlN>Jms6)%B6=``c1I2;8HH9w^DkD{4H_PuHK?# zY_3?V_g;Kxb5uUnW&DaAZTUP~78VPC;7#Kv6@D)+MN5~uup8DKU^|X1P)>!Xpkz`} ziqE-^F38Xxy$`{L1lg6PQ(@PW+|1Jsq}kL5fBP#QMHr(AWa*PE@oPT}#lFsaPigm| zcQ-BDt{XmWC+?Q5h$!CZ8{qJag<|eO*wDC7xzG(?y88f-1G^j?^J4qb#05=W+zuhC z-2f6Jo6CR<(DC z(lp-f->^F4KhJV}93T-k2M>BKi0LS$S?KJ#_y|nae}Oyss7??1wi!2|bt+BwJ8f+7 zclbUp@TVS!Enjj}$)`EJ-5&~YvN__YGp&ZwJ?&B_jGPm(-IYHJ`1X{9%bv{0g_0E_ z%%fz0E9_CXPdHHZdQ=cVp@3;lf#{1u))J}6n=Z5w0x_j;*Qlsy-in_d-=L=SaUYi8 z#m6kAvRZ}FN%7ErZ(ctaIi^@yEb36?C%duKmJYZ5bJtBk)=Gb*#weZRfv3FxutdJc zz$|0tD32|m=vGrH$g?|9Efm1_=XYaeJyv`=1F*-B*6Vp>M5iq63$($714dh9>@}}b z-X5I#2N5KCI{Bo!F*x;SR;E@2Q=i2@*&+cUVY5y}IxD#rCjnwW{I;QTl z8BzMEDVEfD`}|V`y#6QTrUhA7!Vd6DSk-PAHCrebSG~*X=Nph*o-DIjX?r90L2R`8 z_?I2-thq1p4{tE6qS@@ZZ{%=5>@2R>PI6*FhNW`%7!jU=hR#hBs^b!`*^jgQXRlnl zB-`AMUidfo2Ujak?7xmgd@s++6_s*Q{K%Xff_$GLZ2`h#QOb&mC9$ox)@YHs=&Wjd zcbd5Li|6}LM=Rv=o@dXr$Y9ZU`LQmyhfK^p=YG@C_k*A51*peTqhqiQ0c!H(Rg0a^ z2Vkq#im?h z8M*ZVKzms;q{P}g3mkYf;8!TboJ^T3TUcZ(mQM`zZR$2RtuYH#Nvj`s>|Cfc4PDfr zv1PVVX&ev~SFyQJN7aL)T+R?Lv%IO)22_Xd7S@V{^Ir{WX^AV9_;ESSxUh$XMi@Jv z1<~|`JACHiy1q6JIqhlOMRv8*e`e+Q0`L6^FjUhOnq7{ey+GoP+*Iz&SUN*Bb2+*% zNf+^@-Q~0Ss?#hTu?XXU$Iu*c5|8rUZJjK(Ywfo4SYEh14AH*Yb-szxuR#;q-caX1J{YQI%D1Q6Y)-ta=OIp~~ngEt%I)Nnl6LzC+7-hM<$~9cH^PlQ{6ZXb*Z07Lzc;RlJ1wLdC zbCWFLuQO%^e!)Sioc}uYNC9)%wi}pjhqwvDlsxjD`iDIwamU5nuU7G&PGMXLh1w$q zGR_Siv;wa!+cu7yUUDi?(Af)IU4MyvV;WM~ML-+bv8-X73BNIq%MH{o{poWdnonq$j3NJno^;JeQkCop1dX-6lry6Fo4Az**AN z4SywF@%Q|reR+w7lXmBJRLOVR#-G`pjcU-(vebEz-pu8C@q1&IM`BkQ7K3s9Y1-=} z*syI`IUWtuj$J?5+11ODPNYIEvn%gW)(1b-#eW}z)h)NodLJA?x6pE2$o(9!fdiDR zZpo&Wp7F|bkZ}#GK6rkz;4m}oV|Ff`Skm8oDKs;+q~kmKM4!O$E-2O}g1NjoWjXUF zSQWSTxVk=vQ{vM)31PjjD@D>dJ`r6%^K2lp;M<<$6wCl}>V|^13l(Lx-+eZW(@ST( z2jg&@I4KgZwwI$WtJpl%IDK?Ktyq)M(|^fge~Fp3$X8}9F*gm5e-tg`8npQn*|r!* zXt83S6L>pd{%tmj`YX%y&QgaH#ZC)OgFw`+2(pEp3TF742zF007?b7Ji{Oe_7v+bs z`qP6!uuLlp*LXsR}rxXN{2=^RbzB5z}^8K1xnm#T5+LdtTesIajo zU9=U|nDr&;4|Jy+D_T8t@e`sbz;6#nEygf=;?A~jw;WT1+mQ#(%LpInIRMY&CT-@O!6nqO}R3{f0iIUQM6599jbqElLts z9q!y(GB_OFqs{vC1U<2rjl;bbN8U$ndXvRJkP^^z|wOCyg8uNr8%cx4e>j!fjt z0-qSIxPk^+mYx*kmw(vUFY~Cjtcj+IkSBkbyiW4NMJAUC3MC%S zxFlPd9eqo_7 z(+`Lh7K7aNE@QAU@Lj8Bn>;p8p`gAndUW6XADlU5ZU|U+%8b? z!Zx?=VQhC&3BEoJn3T1=jiFGYnItoGaB!WKTKvkzYwOyz3Me^k&d<@t{S9(B;u-!bflP7k> zcFMU{)?0J%$Ybj5-oexN=-`b2k!BVqMa>Kwa8p<&4QP+fp5a<>0}(&>ur! z>R>ZqeaM5h(ej9jvz1M>MV@02XF2Iv%8s+Nj7EOO%lw#oxS^uVr38@Fz zqD9jpV0L~;s*(Q*Thn28=*?C7dD4zEnSXbF)8r@xyAc!jB?Oj2(?f5a=~%kZ_w}kT zP0E!1SOo3ZtIvCsvP~%9C+mmOczW)E&9x@iczrka5NdfD2$1Z-K_4-yKb^I|1wBeX zv(NR{w#OaH3g}g$Dav2PQt0_WuwOm6g^tEzMRbUz#)E4d@1@Vm%Ss^bk9o=JbidSe z<+M;sNm=;MkV-JjSr37M(V_acrPz$K!9N=hsENe~H;gP*{IQ(ULU?d2s5O!_^<9o} z>j$cR8fd3S&L!m7V>ZT4SOZvUn4V|H9-Fs1(I{As8ZFRl>2`8uh&6&Bw~b(a4N>lZ z?XK)yjKZY`ABfF0U!<)TXTy!Zn7ElXjJn>N=luDs9G3Y@_zx*jBePW>Q(c=wE5=yr z0rC5v;DHvc=$Ysf9=jBtGY$8OMhZ-|pkeut{={H1-sUx~*T_D3m%*2= zsKy^43|;FatK=rJ>O7$xLOrb}xJT9BjMR51Zcsgr_Lr3WUd%JBZAw3Bcpx$khjiEQ zln%sd_k?G=lQk?<<9C^BjTtbHZ0!IqL5IL{8)_YBrcl+XfbOGnk z7%wO1CGXX>?=I@Hb+xYIgQ5AYL=!KIAFK}#uw88U156I1wYp_d_ks~bKkZXlNvIj|a`405nL|8N7wL!2$G`kXvrh5*- z9@{j@tf^d=ivgsn3)z+SK%}tJy6bI z9r^@mw!{GT3)qCFCRF5T>FMTU+DS;JkC`#2UY)i+t~4J|6hM;xDDU<*`Sl7j^=Ms4zQ-MgZv%KBHGH=N$W;nx*1}a3FGp-K zLu%h#Tn0(JHkwJxM^dwl(E^K1hNC)8{`D$3f@~)%y@gcN=4ke!sY-OAt7O6Yxd}Wh zxGd-lmu_=59(xf@(xVO!r?$qfP^mO6Yr!XP#SMHx+aE1KU1QMZk zV6JWDj=@JhGrn%KgIqxMQTD-h@2?n&o0k7BNkBUtm?TvE-<=R4_vN% zi;Gwl*4}VJlnI=uaRn&G!k$IU)!Qj9TZNO)_Q<>4^G~QeObXTMBW9)}m}lCv?k1O_ zPOOdjk{uPa>1{nsc^Q9U4E-UqVnRR;IB^`|VywhWl0`P9QO2&=yAx3wb~UYJfb0># zmV0&)zGt~{I8jh2xvWb%d2x8vBUpr7FaUgt@trLa9F0kDR?g+#G;dD^OOa{K;$Z%G z%hRH&(D#0%>9gM3gqvC1ru058r0;4HV??dzUG(LxT-Qx=NqrKj`eiAe*e9~xEr~|I z&@+-M^_Z(Gm^5(ZKFteh8P&749uNxPg9txkHITXyjK9UfhB6}0st$iNL~*gb zJ?(Jlut~zLxb7hR$4RH#P!l1IEYzA4j~=0tUpxakw&3+xYqARQ-EfXXmICTa=Udwd zZ5Ys!tI}j7Pm@BoK)aYtcX}^Jhb7^mhYpT5DU4B*soFKvT1@1yQ%Ym=rB<#uCqnEI z>y5gBJs*bo7Dp6fT{7u}lumECMG%(F7vAFhTtyJTRH_Su9W!p*78iIZ3?pGri#`D7R_F=aTzc_`FC1_{I~Y)`BfY#t64B>f$*bYGbA~* z8krJpY1-fdkta>&nY4|VDm{)GR(Cu6bk3v8i`!*lDtwIs$qROQU+4Imh{*)L0Tu*b zc(ai&BRE#?KLAq#C0_pmcJ@i*ueZ##`yt(*7mJ#Muthl9r(WxG@e9>8HD1E=&Pz&PKonj#^wIOj?sfd_M4p!C;jjZd<_#7CGXBUoN z2P9#(Xy)FPw09=Ox2DXSzR~COp8#cxGCn^zA|&FEN$f1>G(yzZX_I!AkZ?(_O7S4> z^{#8lx)y5Wa4ncum-o*wwumg94MDyc{#7f;`av>pO*l5@b%yQEBsmI~0~d%1WhH#o zJD8bs#9oQ(>}BT*u0dOr3XZf|v7gwBa#|G*ob^l%KLYE> zqJm>-Z&C#Yjk<=7s5#^=o#uHoA9#go^uOXo${(9_c84jwiJ0ih*2&(4J@v zzrJ?lbO*z=aI$BXI+IfSTuh&2>>gUZ5$rZ<+-b@OtLMa|NvvLR0>{yb+wKz7KE_@X zzp3}NdaH7cvwfVdTl##(0R=FtqrlW3%pFg6XzG{L=*>-a-i%g*<^U16O?S;xD~OO# zTB!5$*NFH`*Oom3F@hi`HpoX_s0LW4^9#OyDL!h(?Zkwx^}E3hIsIq5o6oD{c;DSN zvfSR9)_K97=;?r-aoNCR1LgsXA-FppSfEdLyMm<<6#CiAhNS5`$?;&(v&R9p5*#!2 zGnYJY4xR99Y6bV3BQG46Rv`q^7G4mciCL5(e#h|-f4VPZ6%=mLQ>F&?i}hW*kEr9= z)jLBNg%w(!#x4%zk$fFa-}hwhPYZU#rh8xKk64)B42H+zh5U)c_={$zmNWd5 z64C{9C-#Hx6whlq5DA!XOa90s;allKh>n~I3A}vQ zr4L3ktMZ@J8q?+EBk(F5NnjAC@GT|bS2gmRPnU3SoE}3acMoL$NsTiy#>sU5s%u5JB&J#X)=t#l!I80 zmVifeHLB{Pq@pK8)WOh+LNcohXmprvzi8LJ*X44`Z6xGU=7HIh2YvRS*2a2*oMR|RKJERdi( zbKw5c=x72Yy@<-k{n#CB2a^^^3LYW2IV9XeI;Z%w`|m>nR4j}Qk{XYdq!jAvi8)`? zseX+LX3Q-4CTHF3Pk_7WSyp?e9^8MHf|)={u%HD8Hu)+S&qMIHk(BCv2fD z-vJ>eH&xKmTOhES2!u8K-VjDWanbq!9k6@f@NB3%|KMJR0L1c|IzZHRMN4QG_XPG! zp(+BJ7`854YQ>AM>a9;oqXaChR7$4L9>3X)CBin@= z&I&x-1q%GvP9VK3MLD~xAMwTf{jhdyQQz3b3bO z>V3_{4tm=i%dptY63Qs$YxBiW7=cM#d_Ghmsqoh|ROb?ALTW*bV9n5;?`g+v&bc(N1)jvGZ9i$`Hxad3vzvcc)R|5v(lakzp zlt0YS{1d=R7*U|}5S&Y*{mo5z4>v*^FY|cXtP1>>OYmJs!8}5@RQ{{T^i^I7zUp3G zdA2TM`O6FWgIvXuCwXsr*FPz(cn5EPsB@eAf6_G1)5a0~C(G6+IVx9wssEtfAQ3Z{ zT=qc9V;OM&orDUYq|4{TD@{G+^$aQD(U8E>moi^deH7!Z{yAK!fw$ML;Z`r{cOgcu z+L3>~kcEjbN&t#T{6>nOm&l|)t>-`M#U#ygkwRt<9DH)$#8Loe&Fw=<*P+pRUO_Ig` zcf1>Jt~_q-50@A5R}V~89rSr6qfKD$@ojdsdgWg_Uo$iF)_Tj>#hg9mu2Ekh#a#%& zW>j+>a^N72*TYwj@ajpr?JSeJ4w+5DzZLxRu^*=Tvk%6Tf}x;&je0)W$0h97a(q<4 zTql%=O>q&`lNV3n&Rn;H)r6xkxk>bW_us1Je`tSM|F7Hq@C)1l&RRSwNHY0VjSY~I LQj{zcHw^eMW?-Zh diff --git a/public/style.css b/public/style.css deleted file mode 100644 index a2d26d168b..0000000000 --- a/public/style.css +++ /dev/null @@ -1,26 +0,0 @@ -body { - font-family: Helvetica, sans-serif; - padding: 50px; - font-size: 1.1em; -} -input, textarea { - font-family: Helvetica, sans-serif; - padding: 10px; - font-size: 1em; -} -a { - color: #e00; - text-decoration: none; -} -.success { - color: green; -} -.error { - color: red; -} -code { - background-color: #eee; - color: #555; - padding: 0 5px 0 5px; - border-radius: 5px; -} \ No newline at end of file diff --git a/server.js b/recorder.js similarity index 66% rename from server.js rename to recorder.js index 3afb79f030..9a36f0f9a9 100644 --- a/server.js +++ b/recorder.js @@ -5,10 +5,15 @@ try { _ns: 'motley', _maps: [ require('./_codemap'), - require('motley-buffet'), - require('motley-templ') - // require()'ed motley plugins go here. - ] + require('motley-mongo') + ], + // disable http server + '@site.mountMiddleware': function (cb) { + setImmediate(cb) + }, + '@site.listen': function (cb) { + setImmediate(cb) + }, }) } catch (err) { @@ -26,6 +31,7 @@ app.listen(function (err) { function onExit () { app.close(function (err) { if (err) exit(err) + process.exit() }) } process.once('SIGINT', onExit) diff --git a/utils/_codemap.js b/utils/_codemap.js new file mode 100644 index 0000000000..a9d2c9714d --- /dev/null +++ b/utils/_codemap.js @@ -0,0 +1,5 @@ +module.exports = { + _ns: 'motley', + _folder: 'utils', + 'gdaxWebsocket': require('./gdaxWebsocket') +} \ No newline at end of file diff --git a/utils/gdaxWebsocket.js b/utils/gdaxWebsocket.js new file mode 100644 index 0000000000..86adc84ee1 --- /dev/null +++ b/utils/gdaxWebsocket.js @@ -0,0 +1,5 @@ +var CoinbaseExchange = require('coinbase-exchange') + +module.exports = function container (get, set) { + return new CoinbaseExchange.WebsocketClient() +} \ No newline at end of file diff --git a/vendor/_codemap.js b/vendor/_codemap.js deleted file mode 100644 index af3a37ada2..0000000000 --- a/vendor/_codemap.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - // meta - _ns: 'motley', - _folder: 'vendor', - - // vendor repo - 'motley': require('motley') -} \ No newline at end of file diff --git a/views/index.hbs b/views/index.hbs deleted file mode 100644 index 0d71120d09..0000000000 --- a/views/index.hbs +++ /dev/null @@ -1,4 +0,0 @@ -

Welcome {{welcome}}

-

Version: {{version}}

-

Core version: {{core}}

-

Nonce: {{nonce}}

\ No newline at end of file diff --git a/views/layout.hbs b/views/layout.hbs deleted file mode 100644 index 8cbce4a13f..0000000000 --- a/views/layout.hbs +++ /dev/null @@ -1,12 +0,0 @@ - - - - - {{title}} - - - - - {{{content}}} - - \ No newline at end of file From d211f007b92b43a86d3e7e299a83c5ddd6bdc697 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Sun, 26 Jun 2016 21:42:13 -0700 Subject: [PATCH 0002/1659] readme --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index de8cfd1ddc..02d7dc0201 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -# motley-init +# gdax-history -Clone this repo to get started with [Motley node.js framework](https://s8f.org/motley.html). +This thing will stream the [GDAX](https://gdax.com/) order book using the public API and insert records into a `messages` collection in a local MongoDB db called `gdax_history`. -Run "node [server.js](https://github.com/carlos8f/motley-init/blob/master/server.js)" to start the server. +Useful for historical analysis of the order book. -Checkout these other branches for more: +## Usage -- [motley-auth](https://github.com/carlos8f/motley-init/tree/motley-auth) -- [motley-cluster-mongo](https://github.com/carlos8f/motley-init/tree/motley-cluster-mongo) -- [motley-cluster-redis](https://github.com/carlos8f/motley-init/tree/motley-cluster-redis) +``` +$ npm install +$ node recorder.js +``` From 4558d3878e94d165284e84ed279f8e9355cc9313 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 11:49:52 -0700 Subject: [PATCH 0003/1659] reboot socket --- hooks/mountRecorder.js | 25 ++++++++++++++++++++----- package.json | 4 ++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 2b8f242ba5..972467ceda 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -1,10 +1,10 @@ var numeral = require('numeral') -module.exports = function container (get, set) { +module.exports = function container (get, set, clear) { return function mountRecorder (cb) { var socket = get('utils.gdaxWebsocket') var counter = 0 - setInterval(function () { + function onTick () { var totalSize = trades.reduce(function (prev, curr) { return prev + curr.size }, 0) @@ -12,10 +12,19 @@ module.exports = function container (get, set) { return prev + curr.price }, 0) var avg = trades.length ? numeral(totalPrice / trades.length).format('$0,0') : 0 - get('console').log('saved ' + counter + ' messages. trades: ' + trades.length + ' ' + avg + ' / ' + numeral(totalSize).format('0.000')) + var trade_ticker = trades.length ? ' trades: ' + trades.length + ' ' + avg + ' / ' + numeral(totalSize).format('0.000') : '' + get('console').log('saved ' + counter + ' messages.' + trade_ticker) + if (counter === 0) { + get('console').log('no messages in last tick. rebooting socket...') + socket.disconnect() + clear('utils.gdaxWebsocket') + clearInterval(interval) + mountRecorder() + } trades = [] counter = 0 - }, 10000) + } + var interval = setInterval(onTick, 10000) var trades = [] socket.on('message', function (message) { message.id = String(message.sequence) @@ -31,7 +40,13 @@ module.exports = function container (get, set) { } }) }) + socket.on('open', function () { + get('console').log('socket opened.') + }) + socket.on('close', function () { + get('console').log('socket closed.') + }) get('console').log('mounted GDAX recorder.') - cb() + cb && cb() } } \ No newline at end of file diff --git a/package.json b/package.json index 6e7e2ce3c0..6ec30eedf8 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,8 @@ "version": "0.0.0", "description": "", "dependencies": { - "coinbase-exchange": "^0.2.1", - "motley": "^2.2.0", + "coinbase-exchange": "git+https://github.com/carlos8f/coinbase-exchange-node.git", + "motley": "^2.2.7", "motley-buffet": "^2.0.1", "motley-mongo": "^2.0.5", "motley-templ": "^2.0.2", From 82096d96e0336b9015f7b34a2dcf7e263c11d962 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 12:19:17 -0700 Subject: [PATCH 0004/1659] buy/sell indicator --- hooks/mountRecorder.js | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 972467ceda..ed31b4384a 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -5,14 +5,25 @@ module.exports = function container (get, set, clear) { var socket = get('utils.gdaxWebsocket') var counter = 0 function onTick () { - var totalSize = trades.reduce(function (prev, curr) { - return prev + curr.size - }, 0) - var totalPrice = trades.reduce(function (prev, curr) { - return prev + curr.price - }, 0) - var avg = trades.length ? numeral(totalPrice / trades.length).format('$0,0') : 0 - var trade_ticker = trades.length ? ' trades: ' + trades.length + ' ' + avg + ' / ' + numeral(totalSize).format('0.000') : '' + var trade_ticker = '' + if (trades.length) { + var totalSize = trades.reduce(function (prev, curr) { + return prev + curr.size + }, 0) + var totalPrice = trades.reduce(function (prev, curr) { + return prev + curr.price + }, 0) + var numBuy = trades.reduce(function (prev, curr) { + return prev + curr.side === 'buy' ? 1 : 0 + }, 0) + var buyRatio = numBuy / trades.length + var side + if (buyRatio > 0.5) side = 'BUY' + if (buyRatio < 0.5) side = 'SELL' + if (buyRatio === 0.5) side = 'EVEN' + var avg = trades.length ? numeral(totalPrice / trades.length).format('$0,0') : 0 + trade_ticker = trades.length ? ' trades: ' + numBuy + '/' + trades.length + ' ' + side + ' ' + avg + '/' + numeral(totalSize).format('0.000') : '' + } get('console').log('saved ' + counter + ' messages.' + trade_ticker) if (counter === 0) { get('console').log('no messages in last tick. rebooting socket...') @@ -30,12 +41,13 @@ module.exports = function container (get, set, clear) { message.id = String(message.sequence) message.time_date = new Date(message.time) get('db.messages').save(message, function (err, saved) { - if (err) get('console').error('message save err', err) + if (err) return get('console').error('message save err', err) counter++ if (saved.type === 'match' && saved.product_id === 'BTC-USD') { trades.push({ size: parseFloat(saved.size), - price: parseFloat(saved.price) + price: parseFloat(saved.price), + side: saved.side }) } }) From 2ffcddd10e04f73a7cf784377b91c09987dc11aa Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 12:50:18 -0700 Subject: [PATCH 0005/1659] trade ticker fixes --- hooks/mountRecorder.js | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index ed31b4384a..0b18acf3aa 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -1,28 +1,33 @@ var numeral = require('numeral') module.exports = function container (get, set, clear) { + var runningVol = 0, runningTvol = 0 return function mountRecorder (cb) { var socket = get('utils.gdaxWebsocket') var counter = 0 function onTick () { var trade_ticker = '' if (trades.length) { - var totalSize = trades.reduce(function (prev, curr) { - return prev + curr.size - }, 0) - var totalPrice = trades.reduce(function (prev, curr) { - return prev + curr.price - }, 0) - var numBuy = trades.reduce(function (prev, curr) { - return prev + curr.side === 'buy' ? 1 : 0 - }, 0) - var buyRatio = numBuy / trades.length + var high = 0, low = 10000, close, buys = 0, vol = 0, buyVol = 0 + trades.forEach(function (trade) { + high = Math.max(trade.price, high) + low = Math.min(trade.price, low) + close = trade.price + if (trade.side === 'buy') buyVol += trade.size + vol += trade.size + }) + var typical = (high + low + close) / 3 + var tvol = typical * vol + runningVol += vol + runningTvol += tvol + var vwap = runningTvol / runningVol + var buyRatio = buyVol / vol var side if (buyRatio > 0.5) side = 'BUY' if (buyRatio < 0.5) side = 'SELL' if (buyRatio === 0.5) side = 'EVEN' - var avg = trades.length ? numeral(totalPrice / trades.length).format('$0,0') : 0 - trade_ticker = trades.length ? ' trades: ' + numBuy + '/' + trades.length + ' ' + side + ' ' + avg + '/' + numeral(totalSize).format('0.000') : '' + var avg = numeral(vwap).format('$0,0') + trade_ticker = ' trades: ' + numeral(buyRatio).format('(0%)') + ' ' + side + ' ' + avg + '/' + numeral(vol).format('0.000') } get('console').log('saved ' + counter + ' messages.' + trade_ticker) if (counter === 0) { From 4df25c6afd8735dbbc18ec1c09a533a6f6b56d88 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 13:26:24 -0700 Subject: [PATCH 0006/1659] color code trade ticker --- hooks/mountRecorder.js | 9 +++++---- package.json | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 0b18acf3aa..d076dd0891 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -1,4 +1,5 @@ var numeral = require('numeral') + , colors = require('colors') module.exports = function container (get, set, clear) { var runningVol = 0, runningTvol = 0 @@ -23,11 +24,11 @@ module.exports = function container (get, set, clear) { var vwap = runningTvol / runningVol var buyRatio = buyVol / vol var side - if (buyRatio > 0.5) side = 'BUY' - if (buyRatio < 0.5) side = 'SELL' - if (buyRatio === 0.5) side = 'EVEN' + if (buyRatio > 0.5) side = 'BUY'.green + if (buyRatio < 0.5) side = 'SELL'.red + if (buyRatio === 0.5) side = 'EVEN'.orange var avg = numeral(vwap).format('$0,0') - trade_ticker = ' trades: ' + numeral(buyRatio).format('(0%)') + ' ' + side + ' ' + avg + '/' + numeral(vol).format('0.000') + trade_ticker = ' trades: ' + side + ' ' + avg + '/' + numeral(vol).format('0.000') } get('console').log('saved ' + counter + ' messages.' + trade_ticker) if (counter === 0) { diff --git a/package.json b/package.json index 6ec30eedf8..54e53e8d8c 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "", "dependencies": { "coinbase-exchange": "git+https://github.com/carlos8f/coinbase-exchange-node.git", + "colors": "^1.1.2", "motley": "^2.2.7", "motley-buffet": "^2.0.1", "motley-mongo": "^2.0.5", From e09d9f90109726bdee9b03119b2ab9c6683f702f Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 13:49:15 -0700 Subject: [PATCH 0007/1659] fix med color --- hooks/mountRecorder.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index d076dd0891..310635384e 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -24,11 +24,20 @@ module.exports = function container (get, set, clear) { var vwap = runningTvol / runningVol var buyRatio = buyVol / vol var side - if (buyRatio > 0.5) side = 'BUY'.green - if (buyRatio < 0.5) side = 'SELL'.red - if (buyRatio === 0.5) side = 'EVEN'.orange + if (buyRatio > 0.5) side = 'BUY' + if (buyRatio < 0.5) side = 'SELL' + if (buyRatio === 0.5) side = 'EVEN' var avg = numeral(vwap).format('$0,0') trade_ticker = ' trades: ' + side + ' ' + avg + '/' + numeral(vol).format('0.000') + if (vol > 20) { + trade_ticker = trade_ticker.red + } + else if (vol > 5) { + trade_ticker = trade_ticker.yellow + } + else { + trade_ticker = trade_ticker.white + } } get('console').log('saved ' + counter + ' messages.' + trade_ticker) if (counter === 0) { From d59ec658166f57aa911ba40df259ce93deb431e2 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 15:37:51 -0700 Subject: [PATCH 0008/1659] only log trades --- conf/_codemap.js | 7 +- db/_codemap.js | 6 +- db/{messages.js => ticks.js} | 2 +- db/trades.js | 20 +++++ hooks/ensureIndexes.js | 18 +++++ hooks/mountRecorder.js | 144 +++++++++++++++++++++++------------ package.json | 3 +- 7 files changed, 145 insertions(+), 55 deletions(-) rename db/{messages.js => ticks.js} (90%) create mode 100644 db/trades.js create mode 100644 hooks/ensureIndexes.js diff --git a/conf/_codemap.js b/conf/_codemap.js index fcd1f462a6..27a30f4d87 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -7,5 +7,10 @@ module.exports = { url: 'mongodb://localhost:27017/gdax_history', username: null, password: null - } + }, + + 'product_id': 'BTC-USD', + + 'tick_interval': 10000, + 'tick_size': '10s' } \ No newline at end of file diff --git a/db/_codemap.js b/db/_codemap.js index 0c55b31bc9..7ce0196c06 100644 --- a/db/_codemap.js +++ b/db/_codemap.js @@ -4,11 +4,13 @@ module.exports = { _folder: 'db', // named collections - 'messages': require('./messages'), + 'ticks': require('./ticks'), + 'trades': require('./trades'), // collection registration 'collections[]': [ - '#db.messages' + '#db.ticks', + '#db.trades' // add more collections here. ] } \ No newline at end of file diff --git a/db/messages.js b/db/ticks.js similarity index 90% rename from db/messages.js rename to db/ticks.js index 1a91ad03d8..d2d99755b3 100644 --- a/db/messages.js +++ b/db/ticks.js @@ -1,5 +1,5 @@ module.exports = function container (get, set) { - return get('db.createCollection')('messages', { + return get('db.createCollection')('ticks', { load: function (obj, opts, cb) { // respond after the obj is loaded cb(null, obj); diff --git a/db/trades.js b/db/trades.js new file mode 100644 index 0000000000..65211d56d6 --- /dev/null +++ b/db/trades.js @@ -0,0 +1,20 @@ +module.exports = function container (get, set) { + return get('db.createCollection')('trades', { + load: function (obj, opts, cb) { + // respond after the obj is loaded + cb(null, obj); + }, + save: function (obj, opts, cb) { + // respond before the obj is saved + cb(null, obj); + }, + afterSave: function (obj, opts, cb) { + // respond after the obj is saved + cb(null, obj); + }, + destroy: function (obj, opts, cb) { + // respond after the obj is destroyed + cb(null, obj) + } + }) +} \ No newline at end of file diff --git a/hooks/ensureIndexes.js b/hooks/ensureIndexes.js new file mode 100644 index 0000000000..08bb09851b --- /dev/null +++ b/hooks/ensureIndexes.js @@ -0,0 +1,18 @@ +module.exports = function container (get, set) { + return function ensureIndexes (cb) { + var tasks = [] + tasks.push(function (done) { + get('db.mongo.db').collection('trades').ensureIndex({time: -1}, done) + }) + tasks.push(function (done) { + get('db.mongo.db').collection('trades').ensureIndex({time: 1}, done) + }) + tasks.push(function (done) { + get('db.mongo.db').collection('ticks').ensureIndex({time: -1}, done) + }) + tasks.push(function (done) { + get('db.mongo.db').collection('ticks').ensureIndex({time: 1}, done) + }) + get('vendor.run-series')(tasks, cb) + } +} \ No newline at end of file diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 310635384e..cd9a00f4a9 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -1,5 +1,6 @@ var numeral = require('numeral') , colors = require('colors') + , tb = require('timebucket') module.exports = function container (get, set, clear) { var runningVol = 0, runningTvol = 0 @@ -8,64 +9,107 @@ module.exports = function container (get, set, clear) { var counter = 0 function onTick () { var trade_ticker = '' - if (trades.length) { - var high = 0, low = 10000, close, buys = 0, vol = 0, buyVol = 0 - trades.forEach(function (trade) { - high = Math.max(trade.price, high) - low = Math.min(trade.price, low) - close = trade.price - if (trade.side === 'buy') buyVol += trade.size - vol += trade.size - }) - var typical = (high + low + close) / 3 - var tvol = typical * vol - runningVol += vol - runningTvol += tvol - var vwap = runningTvol / runningVol - var buyRatio = buyVol / vol - var side - if (buyRatio > 0.5) side = 'BUY' - if (buyRatio < 0.5) side = 'SELL' - if (buyRatio === 0.5) side = 'EVEN' - var avg = numeral(vwap).format('$0,0') - trade_ticker = ' trades: ' + side + ' ' + avg + '/' + numeral(vol).format('0.000') - if (vol > 20) { - trade_ticker = trade_ticker.red + var params = { + query: { + time: { + $gt: new Date().getTime() - get('conf.tick_interval') + } + }, + sort: { + time: 1 } - else if (vol > 5) { - trade_ticker = trade_ticker.yellow + } + get('db.trades').select(params, function (err, trades) { + if (err) return get('console').error('trade select err', err) + if (trades.length) { + var high = 0, low = 10000, close, buys = 0, vol = 0, buyVol = 0 + trades.forEach(function (trade) { + high = Math.max(trade.price, high) + low = Math.min(trade.price, low) + close = trade.price + if (trade.side === 'sell') { + buyVol += trade.size + buys++ + } + vol += trade.size + }) + var typical = (high + low + close) / 3 + var tvol = typical * vol + runningVol += vol + runningTvol += tvol + var vwap = runningTvol / runningVol + var buyRatio = buyVol / vol + var side + if (buyRatio > 0.5) side = 'BUY' + if (buyRatio < 0.5) side = 'SELL' + if (buyRatio === 0.5) side = 'EVEN' + var avg = numeral(vwap).format('$0,0') + trade_ticker = side + ' ' + avg + '/' + numeral(vol).format('0.000') + var orig_ticker = trade_ticker + if (vol > 20) { + trade_ticker = trade_ticker.red + } + else if (vol > 5) { + trade_ticker = trade_ticker.yellow + } + else { + trade_ticker = trade_ticker.white + } + trade_ticker = ' trades: ' + trade_ticker + var tick = { + id: tb(get('conf.tick_size')).toString(), + time: new Date().getTime(), + date: new Date(), + vol: vol, + high: high, + low: low, + close: close, + trades: trades.length, + buys: buys, + buyVol: buyVol, + vwap: vwap, + buyRatio: buyRatio, + typical: typical, + side: side, + avg: avg, + ticker: orig_ticker + } + get('db.ticks').save(tick, function (err, saved) { + if (err) return get('console').error('tick save err', err) + }) } else { - trade_ticker = trade_ticker.white + get('db.ticks').save({id: tb(get('conf.tick_size')).toString(), vol: 0, time: new Date().getTime(), date: new Date()}, function (err, saved) { + if (err) return get('console').error('tick save err', err) + }) } - } - get('console').log('saved ' + counter + ' messages.' + trade_ticker) - if (counter === 0) { - get('console').log('no messages in last tick. rebooting socket...') - socket.disconnect() - clear('utils.gdaxWebsocket') - clearInterval(interval) - mountRecorder() - } - trades = [] - counter = 0 + get('console').log('saw ' + counter + ' messages.' + trade_ticker) + if (counter === 0) { + get('console').log('no messages in last tick. rebooting socket...') + socket.disconnect() + clear('utils.gdaxWebsocket') + clearInterval(interval) + mountRecorder() + } + counter = 0 + }) } - var interval = setInterval(onTick, 10000) + var interval = setInterval(onTick, get('conf.tick_interval')) var trades = [] socket.on('message', function (message) { - message.id = String(message.sequence) - message.time_date = new Date(message.time) - get('db.messages').save(message, function (err, saved) { - if (err) return get('console').error('message save err', err) - counter++ - if (saved.type === 'match' && saved.product_id === 'BTC-USD') { - trades.push({ - size: parseFloat(saved.size), - price: parseFloat(saved.price), - side: saved.side - }) + counter++ + if (message.type === 'match' && message.product_id === get('conf.product_id')) { + var trade = { + id: String(message.sequence), + time: new Date(message.time).getTime(), + size: parseFloat(message.size), + price: parseFloat(message.price), + side: message.side } - }) + get('db.trades').save(trade, function (err, saved) { + if (err) return get('console').error('trade save err', err) + }) + } }) socket.on('open', function () { get('console').log('socket opened.') diff --git a/package.json b/package.json index 54e53e8d8c..fc36f901e8 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "motley-buffet": "^2.0.1", "motley-mongo": "^2.0.5", "motley-templ": "^2.0.2", - "numeral": "^1.5.3" + "numeral": "^1.5.3", + "timebucket": "^0.3.4" } } From 0843e1c96b51b50300466129460724b1e102a774 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 22:04:20 -0700 Subject: [PATCH 0009/1659] backfiller and bot --- backfiller.js | 36 +++++++++++++ bot.js | 36 +++++++++++++ conf/_codemap.js | 11 +++- db/ticks.js | 65 +++++++++++++++++++++++ hooks/_codemap.js | 5 +- hooks/mountBackfiller.js | 47 +++++++++++++++++ hooks/mountBot.js | 110 +++++++++++++++++++++++++++++++++++++++ hooks/mountRecorder.js | 67 +----------------------- package.json | 1 + recorder.js | 13 ++--- utils/_codemap.js | 1 + utils/gdaxClient.js | 5 ++ 12 files changed, 322 insertions(+), 75 deletions(-) create mode 100644 backfiller.js create mode 100644 bot.js create mode 100644 hooks/mountBackfiller.js create mode 100644 hooks/mountBot.js create mode 100644 utils/gdaxClient.js diff --git a/backfiller.js b/backfiller.js new file mode 100644 index 0000000000..361507dd56 --- /dev/null +++ b/backfiller.js @@ -0,0 +1,36 @@ +var motley = require('motley') + +try { + var app = motley({ + _ns: 'motley', + _maps: [ + require('./_codemap'), + require('motley-mongo') + ], + '@hooks.mount': [ + '#db.mountCollections', + '#hooks.mountBackfiller' + ] + }) +} +catch (err) { + exit(err) +} + +function exit (err) { + console.error(err) + console.error(err.stack) + process.exit(1) +} + +app.mount(function (err) { + if (err) exit(err) + function onExit () { + app.close(function (err) { + if (err) exit(err) + process.exit() + }) + } + process.once('SIGINT', onExit) + process.once('SIGTERM', onExit) +}) \ No newline at end of file diff --git a/bot.js b/bot.js new file mode 100644 index 0000000000..6fc0505f7c --- /dev/null +++ b/bot.js @@ -0,0 +1,36 @@ +var motley = require('motley') + +try { + var app = motley({ + _ns: 'motley', + _maps: [ + require('./_codemap'), + require('motley-mongo') + ], + '@hooks.mount': [ + '#db.mountCollections', + '#hooks.mountBot' + ] + }) +} +catch (err) { + exit(err) +} + +function exit (err) { + console.error(err) + console.error(err.stack) + process.exit(1) +} + +app.mount(function (err) { + if (err) exit(err) + function onExit () { + app.close(function (err) { + if (err) exit(err) + process.exit() + }) + } + process.once('SIGINT', onExit) + process.once('SIGTERM', onExit) +}) \ No newline at end of file diff --git a/conf/_codemap.js b/conf/_codemap.js index 27a30f4d87..f3b8b0f727 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -12,5 +12,14 @@ module.exports = { 'product_id': 'BTC-USD', 'tick_interval': 10000, - 'tick_size': '10s' + 'tick_size': '10s', + + 'bot': { + balance: { + asset: 0, + currency: 1000 + }, + min_vol: 140, + min_trade: 0.01 + } } \ No newline at end of file diff --git a/db/ticks.js b/db/ticks.js index d2d99755b3..0f1dfa2c71 100644 --- a/db/ticks.js +++ b/db/ticks.js @@ -1,3 +1,7 @@ +var numeral = require('numeral') + , colors = require('colors') + , tb = require('timebucket') + module.exports = function container (get, set) { return get('db.createCollection')('ticks', { load: function (obj, opts, cb) { @@ -15,6 +19,67 @@ module.exports = function container (get, set) { destroy: function (obj, opts, cb) { // respond after the obj is destroyed cb(null, obj) + }, + methods: { + create: function (trades) { + var trade_ticker = '' + if (trades.length) { + var open, high = 0, low = 10000, close, buys = 0, vol = 0, buyVol = 0 + var closeTime + trades.forEach(function (trade) { + if (typeof open === 'undefined') open = trade.price + high = Math.max(trade.price, high) + low = Math.min(trade.price, low) + close = trade.price + closeTime = trade.time + if (trade.side === 'sell') { + buyVol += trade.size + buys++ + } + vol += trade.size + }) + var typical = (high + low + close) / 3 + var buyRatio = buyVol / vol + var side + if (buyRatio > 0.5) side = 'BUY' + if (buyRatio < 0.5) side = 'SELL' + if (buyRatio === 0.5) side = 'EVEN' + var bucket = tb(closeTime).resize(get('conf.tick_size')) + var tick = { + id: bucket.toString(), + time: bucket.toMilliseconds(), + vol: vol, + high: high, + low: low, + open: open, + close: close, + trades: trades.length, + buys: buys, + buyVol: buyVol, + buyRatio: buyRatio, + typical: typical, + price: numeral(typical).format('$0,0.00'), + side: side, + ticker: orig_ticker + } + trade_ticker = side + ' ' + tick.price + '/' + numeral(vol).format('0.000') + var orig_ticker = trade_ticker + if (vol > 20) { + trade_ticker = trade_ticker.red + } + else if (vol > 5) { + trade_ticker = trade_ticker.yellow + } + else { + trade_ticker = trade_ticker.white + } + trade_ticker = ' trades: ' + trade_ticker + get('db.ticks').save(tick, function (err, saved) { + if (err) return get('console').error('tick save err', err) + }) + } + return trade_ticker + } } }) } \ No newline at end of file diff --git a/hooks/_codemap.js b/hooks/_codemap.js index b7157a3f17..8e378cf009 100644 --- a/hooks/_codemap.js +++ b/hooks/_codemap.js @@ -2,5 +2,8 @@ module.exports = { // meta _ns: 'motley', _folder: 'hooks', - 'mount[]': require('./mountRecorder') + + 'mountBackfiller': require('./mountBackfiller'), + 'mountBot': require('./mountBot'), + 'mountRecorder': require('./mountRecorder') } \ No newline at end of file diff --git a/hooks/mountBackfiller.js b/hooks/mountBackfiller.js new file mode 100644 index 0000000000..517796db4f --- /dev/null +++ b/hooks/mountBackfiller.js @@ -0,0 +1,47 @@ +var numeral = require('numeral') + , colors = require('colors') + , tb = require('timebucket') + +module.exports = function container (get, set, clear) { + return function mountBackfiller (cb) { + var client = get('utils.gdaxClient') + var counter = 0 + var after = undefined + function getNext () { + client.getProductTrades({after: after}, function (err, resp, trades) { + if (err) { + get('console').error('trade fetch err', err) + return setTimeout(getNext, 5000) + } + if (!trades) return get('console').error('no trades error') + if (!trades.length) return get('console').log('backfilling done!?') + var trades = trades.map(function (trade) { + return { + id: String(trade.trade_id), + time: new Date(trade.time).getTime(), + size: parseFloat(trade.size), + price: parseFloat(trade.price), + side: trade.side + } + }).reverse() + after = trades[0].id + var ticks = {} + trades.forEach(function (trade) { + var tickId = tb(trade.time).resize(get('conf.tick_size')).toString() + ticks[tickId] || (ticks[tickId] = []) + ticks[tickId].push(trade) + counter++ + }) + Object.keys(ticks).forEach(function (tickId) { + var ticker = get('db.ticks').create(ticks[tickId]) + if (ticker) get('console').log('backfilled', tb(tickId).toDate(), ticker) + }) + get('console').log('processed', counter, 'trades. after = ' + after) + setTimeout(getNext, 0) + }) + } + setTimeout(getNext, 1000) + get('console').log('mounted GDAX backfiller.') + cb && cb() + } +} \ No newline at end of file diff --git a/hooks/mountBot.js b/hooks/mountBot.js new file mode 100644 index 0000000000..645e55ff99 --- /dev/null +++ b/hooks/mountBot.js @@ -0,0 +1,110 @@ +var numeral = require('numeral') + , colors = require('colors') + , tb = require('timebucket') + , through = require('through') + +module.exports = function container (get, set, clear) { + return function mountBot (cb) { + var minTime = new Date().getTime() - (86400000 * 90) // 90 days ago + var bot = get('conf.bot') + var initBalance = JSON.parse(JSON.stringify(bot.balance)) + var side = null + var close = 0 + var vol = 0 + var counter = 0 + + function printReport () { + var newBalance = JSON.parse(JSON.stringify(bot.balance)) + newBalance.currency += newBalance.asset * close + newBalance.asset = 0 + var diff = newBalance.currency - initBalance.currency + if (diff > 0) diff = ('+' + numeral(diff).format('$0,0.00')).green + if (diff === 0) diff = ('+' + numeral(diff).format('$0,0.00')).white + if (diff < 0) diff = (numeral(diff).format('$0,0.00')).red + get('console').log('[bot]', diff, numeral(bot.balance.asset).format('0.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) + } + + var tickStream = through(function write (tick) { + if (side && tick.side !== side) { + vol -= tick.vol + if (vol < 0) side = tick.side + vol = Math.abs(vol) + } + else { + side = tick.side + vol += tick.vol + } + if (vol >= bot.min_vol) { + vol = 0 + // trigger + if (side === 'BUY' && !bot.balance.currency) { + get('console').log('got BUY signal but i\'m broke.') + } + else if (side === 'SELL' && !bot.balance.asset) { + get('console').log('got SELL signal but i got no BTC') + } + else if (side === 'BUY') { + var spend = bot.balance.currency / 2 + if (spend / close < bot.min_trade) { + get('console').log('[bot] HOLD') + return + } + bot.balance.currency -= spend + bot.balance.asset += spend / close + get('console').log('[bot] BUY ' + numeral(spend / close).format('0.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + } + else if (side === 'SELL') { + var sell = bot.balance.asset / 2 + if (sell < bot.min_trade) { + get('console').log('[bot] HOLD') + return + } + bot.balance.asset -= sell + bot.balance.currency += sell * close + get('console').log('[bot] SELL ' + numeral(sell).format('0.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + } + printReport() + } + }) + function getNext () { + var params = { + query: { + time: { + $gt: minTime + } + }, + sort: { + time: 1 + }, + limit: 100 + } + get('db.ticks').select(params, function (err, ticks) { + if (err) { + get('console').error('tick select err', err) + return setTimeout(getNext, 1000) + } + if (!ticks.length) { + return setTimeout(getNext, get('conf.tick_interval')) + } + ticks.forEach(function (tick) { + if (!close) { + initBalance.currency += initBalance.asset * tick.close + initBalance.asset = 0 + } + close = tick.close + tickStream.write(tick) + minTime = tick.time + counter++ + }) + var date = new Date(minTime) + var tzMatch = date.toString().match(/\((.*)\)/) + var time = date.toLocaleString() + ' ' + tzMatch[1] + get('console').log(numeral(close).format('$0,0.00'), time.grey, numeral(bot.balance.asset).format('0.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) + getNext() + }) + } + setTimeout(getNext, 1000) + get('console').log('mounted bot.') + cb && cb() + } +} \ No newline at end of file diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index cd9a00f4a9..58ef38b19d 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -3,7 +3,6 @@ var numeral = require('numeral') , tb = require('timebucket') module.exports = function container (get, set, clear) { - var runningVol = 0, runningTvol = 0 return function mountRecorder (cb) { var socket = get('utils.gdaxWebsocket') var counter = 0 @@ -21,69 +20,8 @@ module.exports = function container (get, set, clear) { } get('db.trades').select(params, function (err, trades) { if (err) return get('console').error('trade select err', err) - if (trades.length) { - var high = 0, low = 10000, close, buys = 0, vol = 0, buyVol = 0 - trades.forEach(function (trade) { - high = Math.max(trade.price, high) - low = Math.min(trade.price, low) - close = trade.price - if (trade.side === 'sell') { - buyVol += trade.size - buys++ - } - vol += trade.size - }) - var typical = (high + low + close) / 3 - var tvol = typical * vol - runningVol += vol - runningTvol += tvol - var vwap = runningTvol / runningVol - var buyRatio = buyVol / vol - var side - if (buyRatio > 0.5) side = 'BUY' - if (buyRatio < 0.5) side = 'SELL' - if (buyRatio === 0.5) side = 'EVEN' - var avg = numeral(vwap).format('$0,0') - trade_ticker = side + ' ' + avg + '/' + numeral(vol).format('0.000') - var orig_ticker = trade_ticker - if (vol > 20) { - trade_ticker = trade_ticker.red - } - else if (vol > 5) { - trade_ticker = trade_ticker.yellow - } - else { - trade_ticker = trade_ticker.white - } - trade_ticker = ' trades: ' + trade_ticker - var tick = { - id: tb(get('conf.tick_size')).toString(), - time: new Date().getTime(), - date: new Date(), - vol: vol, - high: high, - low: low, - close: close, - trades: trades.length, - buys: buys, - buyVol: buyVol, - vwap: vwap, - buyRatio: buyRatio, - typical: typical, - side: side, - avg: avg, - ticker: orig_ticker - } - get('db.ticks').save(tick, function (err, saved) { - if (err) return get('console').error('tick save err', err) - }) - } - else { - get('db.ticks').save({id: tb(get('conf.tick_size')).toString(), vol: 0, time: new Date().getTime(), date: new Date()}, function (err, saved) { - if (err) return get('console').error('tick save err', err) - }) - } - get('console').log('saw ' + counter + ' messages.' + trade_ticker) + var ticker = get('db.ticks').create(trades) + get('console').log('saw ' + counter + ' messages.' + ticker) if (counter === 0) { get('console').log('no messages in last tick. rebooting socket...') socket.disconnect() @@ -95,7 +33,6 @@ module.exports = function container (get, set, clear) { }) } var interval = setInterval(onTick, get('conf.tick_interval')) - var trades = [] socket.on('message', function (message) { counter++ if (message.type === 'match' && message.product_id === get('conf.product_id')) { diff --git a/package.json b/package.json index fc36f901e8..9dc693fa53 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "motley-mongo": "^2.0.5", "motley-templ": "^2.0.2", "numeral": "^1.5.3", + "through": "^2.3.8", "timebucket": "^0.3.4" } } diff --git a/recorder.js b/recorder.js index 9a36f0f9a9..c3b51debe0 100644 --- a/recorder.js +++ b/recorder.js @@ -7,13 +7,10 @@ try { require('./_codemap'), require('motley-mongo') ], - // disable http server - '@site.mountMiddleware': function (cb) { - setImmediate(cb) - }, - '@site.listen': function (cb) { - setImmediate(cb) - }, + '@hooks.mount': [ + '#db.mountCollections', + '#hooks.mountRecorder' + ] }) } catch (err) { @@ -26,7 +23,7 @@ function exit (err) { process.exit(1) } -app.listen(function (err) { +app.mount(function (err) { if (err) exit(err) function onExit () { app.close(function (err) { diff --git a/utils/_codemap.js b/utils/_codemap.js index a9d2c9714d..e222dfc0d5 100644 --- a/utils/_codemap.js +++ b/utils/_codemap.js @@ -1,5 +1,6 @@ module.exports = { _ns: 'motley', _folder: 'utils', + 'gdaxClient': require('./gdaxClient'), 'gdaxWebsocket': require('./gdaxWebsocket') } \ No newline at end of file diff --git a/utils/gdaxClient.js b/utils/gdaxClient.js new file mode 100644 index 0000000000..8ea75e666d --- /dev/null +++ b/utils/gdaxClient.js @@ -0,0 +1,5 @@ +var CoinbaseExchange = require('coinbase-exchange') + +module.exports = function container (get, set) { + return new CoinbaseExchange.PublicClient() +} \ No newline at end of file From b27b489f14767450c5b79d47664dd363db2f0d0c Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 22:50:39 -0700 Subject: [PATCH 0010/1659] graph --- hooks/mountBot.js | 58 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/hooks/mountBot.js b/hooks/mountBot.js index 645e55ff99..63737095c4 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -9,9 +9,11 @@ module.exports = function container (get, set, clear) { var bot = get('conf.bot') var initBalance = JSON.parse(JSON.stringify(bot.balance)) var side = null - var close = 0 - var vol = 0 + var periodVol = 0 var counter = 0 + var runningVol = 0, runningTotal = 0 + var high = 0, low = 10000, close = 0, vol = 0 + var maxDiff = 0 function printReport () { var newBalance = JSON.parse(JSON.stringify(bot.balance)) @@ -21,10 +23,50 @@ module.exports = function container (get, set, clear) { if (diff > 0) diff = ('+' + numeral(diff).format('$0,0.00')).green if (diff === 0) diff = ('+' + numeral(diff).format('$0,0.00')).white if (diff < 0) diff = (numeral(diff).format('$0,0.00')).red - get('console').log('[bot]', diff, numeral(bot.balance.asset).format('0.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) + get('console').log('[bot]', diff) + } + + function getGraph () { + runningTotal += ((high + low + close) / 3) * periodVol + //console.log('runningTotal', runningTotal) + runningVol += periodVol + //console.log('runningVol', runningVol) + var vwap = runningTotal / runningVol + //console.log('vwap', vwap) + var vwapDiff = close - vwap + //console.log('vwapDiff', vwapDiff) + maxDiff = Math.max(maxDiff, Math.abs(vwapDiff)) + //console.log('maxDiff', maxDiff) + var barWidth = 20 + var half = barWidth / 2 + var bar = '' + if (vwapDiff > 0) { + bar += ' '.repeat(half) + var stars = Math.min(Math.round((vwapDiff / maxDiff) * half), half) + bar += '+'.green.repeat(stars) + bar += ' '.repeat(half - stars) + } + else if (vwapDiff < 0) { + var stars = Math.min(Math.round((Math.abs(vwapDiff) / maxDiff) * half), half) + bar += ' '.repeat(half - stars) + bar += '-'.red.repeat(stars) + bar += ' '.repeat(half) + } + else { + bar += ' '.repeat(half * 2) + } + vol = 0 + high = 0 + low = 10000 + return bar } var tickStream = through(function write (tick) { + periodVol += tick.vol + close = tick.close + high = Math.max(high, tick.high) + low = Math.min(low, tick.low) + if (side && tick.side !== side) { vol -= tick.vol if (vol < 0) side = tick.side @@ -51,7 +93,7 @@ module.exports = function container (get, set, clear) { } bot.balance.currency -= spend bot.balance.asset += spend / close - get('console').log('[bot] BUY ' + numeral(spend / close).format('0.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + get('console').log('[bot] BUY ' + numeral(spend / close).format('00.000') + ' BTC at ' + numeral(close).format('$0,0.00')) } else if (side === 'SELL') { var sell = bot.balance.asset / 2 @@ -61,7 +103,7 @@ module.exports = function container (get, set, clear) { } bot.balance.asset -= sell bot.balance.currency += sell * close - get('console').log('[bot] SELL ' + numeral(sell).format('0.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(close).format('$0,0.00')) } printReport() } @@ -99,7 +141,11 @@ module.exports = function container (get, set, clear) { var date = new Date(minTime) var tzMatch = date.toString().match(/\((.*)\)/) var time = date.toLocaleString() + ' ' + tzMatch[1] - get('console').log(numeral(close).format('$0,0.00'), time.grey, numeral(bot.balance.asset).format('0.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) + if (time.length === 24) { + time = time.replace(', ', ', 0') + } + var bar = getGraph() + get('console').log(bar + ' ' + numeral(close).format('$0,0.00'), time.grey, numeral(bot.balance.asset).format('00.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) getNext() }) } From 92f67809f47e99311aa226ca1c29893f3b65880a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 22:52:22 -0700 Subject: [PATCH 0011/1659] silly --- hooks/mountBot.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hooks/mountBot.js b/hooks/mountBot.js index 63737095c4..2e4cd42aef 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -80,10 +80,12 @@ module.exports = function container (get, set, clear) { vol = 0 // trigger if (side === 'BUY' && !bot.balance.currency) { - get('console').log('got BUY signal but i\'m broke.') + get('console').log('[bot] HOLD') + return } else if (side === 'SELL' && !bot.balance.asset) { - get('console').log('got SELL signal but i got no BTC') + get('console').log('[bot] HOLD') + return } else if (side === 'BUY') { var spend = bot.balance.currency / 2 From 0281dab4d029b209eba0f638440965e29021e212 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 27 Jun 2016 23:30:36 -0700 Subject: [PATCH 0012/1659] tweaks --- README.md | 28 +++++++++++++++++++++++++--- conf/_codemap.js | 9 ++++++++- hooks/mountBot.js | 25 ++++++++++++++++--------- package.json | 6 ++---- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 02d7dc0201..8a7c6fc5d1 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,34 @@ -# gdax-history +# zenbot -This thing will stream the [GDAX](https://gdax.com/) order book using the public API and insert records into a `messages` collection in a local MongoDB db called `gdax_history`. +# !ALPHA SOFTWARE. DO NOT USE FOR REAL TRADES. -Useful for historical analysis of the order book. +zenbot is a passive trading bot for [GDAX](https://gdax.com/). zen trusts other bots. ## Usage +Configure it first in [conf/_codemap.js](https://github.com/carlos8f/zenbot/blob/master/conf/_codemap.js) + +## record trades + ``` $ npm install $ node recorder.js ``` + +## backfill trades + +``` +$ node backfiller.js +``` + +### run trade simulation (zen mode) + +``` +$ node bot.js --sim +``` + +### run trade bot (zen+ mode) + +``` +$ node bot.js +``` diff --git a/conf/_codemap.js b/conf/_codemap.js index f3b8b0f727..b1c5796f4c 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -14,12 +14,19 @@ module.exports = { 'tick_interval': 10000, 'tick_size': '10s', + 'gdax': require('../gdax-config.js'), + 'bot': { balance: { asset: 0, currency: 1000 }, min_vol: 140, - min_trade: 0.01 + min_trade: 0.01, + sim: process.argv[2] !== '--real', + markup: 0.0002, + fee: 0.0025, + query_limit: 100, + crash_protection: 0.0019 } } \ No newline at end of file diff --git a/hooks/mountBot.js b/hooks/mountBot.js index 2e4cd42aef..f03365b82f 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -89,13 +89,17 @@ module.exports = function container (get, set, clear) { } else if (side === 'BUY') { var spend = bot.balance.currency / 2 - if (spend / close < bot.min_trade) { + var price = close + (close * bot.markup) // add markup + if (spend / price < bot.min_trade) { get('console').log('[bot] HOLD') return } bot.balance.currency -= spend - bot.balance.asset += spend / close - get('console').log('[bot] BUY ' + numeral(spend / close).format('00.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + var size = spend / price + bot.balance.asset += size + var fee = (size * price) * bot.fee + bot.balance.currency -= fee + get('console').log('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00')) } else if (side === 'SELL') { var sell = bot.balance.asset / 2 @@ -104,8 +108,11 @@ module.exports = function container (get, set, clear) { return } bot.balance.asset -= sell - bot.balance.currency += sell * close - get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(close).format('$0,0.00')) + var price = close - (close * bot.markup) + bot.balance.currency += sell * price + var fee = (sell * price) * bot.fee + bot.balance.currency -= fee + get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00')) } printReport() } @@ -125,7 +132,7 @@ module.exports = function container (get, set, clear) { get('db.ticks').select(params, function (err, ticks) { if (err) { get('console').error('tick select err', err) - return setTimeout(getNext, 1000) + return setImmediate(getNext) } if (!ticks.length) { return setTimeout(getNext, get('conf.tick_interval')) @@ -143,15 +150,15 @@ module.exports = function container (get, set, clear) { var date = new Date(minTime) var tzMatch = date.toString().match(/\((.*)\)/) var time = date.toLocaleString() + ' ' + tzMatch[1] - if (time.length === 24) { + if (time.match(/, [^0]:/)) { time = time.replace(', ', ', 0') } var bar = getGraph() get('console').log(bar + ' ' + numeral(close).format('$0,0.00'), time.grey, numeral(bot.balance.asset).format('00.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) - getNext() + setImmediate(getNext) }) } - setTimeout(getNext, 1000) + setImmediate(getNext) get('console').log('mounted bot.') cb && cb() } diff --git a/package.json b/package.json index 9dc693fa53..be1ccecda3 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,12 @@ { - "name": "your-new-site", + "name": "zenbot", "version": "0.0.0", - "description": "", + "description": "a passive trading bot for gdax (coinbase exchange)", "dependencies": { "coinbase-exchange": "git+https://github.com/carlos8f/coinbase-exchange-node.git", "colors": "^1.1.2", "motley": "^2.2.7", - "motley-buffet": "^2.0.1", "motley-mongo": "^2.0.5", - "motley-templ": "^2.0.2", "numeral": "^1.5.3", "through": "^2.3.8", "timebucket": "^0.3.4" From 564a9d294e869a99eb4d27bb6442907100dc2f30 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 10:14:42 -0700 Subject: [PATCH 0013/1659] crash protection --- .gitignore | 1 + README.md | 8 ++++---- hooks/mountBot.js | 32 ++++++++++++++++++++++---------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 12faf8db25..fda7f4ca7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules npm-debug.log db.json +gdax-config.js \ No newline at end of file diff --git a/README.md b/README.md index 8a7c6fc5d1..f059fa606c 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ $ node recorder.js $ node backfiller.js ``` -### run trade simulation (zen mode) +### run trade simulation ``` -$ node bot.js --sim +$ node bot.js ``` -### run trade bot (zen+ mode) +### run trade bot and do ACTUAL trades (zen mode) ``` -$ node bot.js +$ node bot.js --real ``` diff --git a/hooks/mountBot.js b/hooks/mountBot.js index f03365b82f..2a0c3e9dca 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -12,7 +12,7 @@ module.exports = function container (get, set, clear) { var periodVol = 0 var counter = 0 var runningVol = 0, runningTotal = 0 - var high = 0, low = 10000, close = 0, vol = 0 + var high = 0, low = 10000, close = 0, vol = 0, lastClose = 0 var maxDiff = 0 function printReport () { @@ -88,10 +88,15 @@ module.exports = function container (get, set, clear) { return } else if (side === 'BUY') { - var spend = bot.balance.currency / 2 + var delta = 1 - (lastClose / close) var price = close + (close * bot.markup) // add markup + if (delta >= bot.crash_protection) { + get('console').log('[bot] refusing to BUY at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.0%')) + return + } + var spend = bot.balance.currency / 2 if (spend / price < bot.min_trade) { - get('console').log('[bot] HOLD') + get('console').log('[bot] HOLD', numeral(delta).format('0.0%')) return } bot.balance.currency -= spend @@ -99,23 +104,29 @@ module.exports = function container (get, set, clear) { bot.balance.asset += size var fee = (size * price) * bot.fee bot.balance.currency -= fee - get('console').log('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00')) + get('console').log('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00'), numeral(delta).format('0.0%')) } else if (side === 'SELL') { + var price = close - (close * bot.markup) // add markup + var delta = 1 - (close / lastClose) + if (delta >= bot.crash_protection) { + get('console').log('[bot] refusing to SELL at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.0%')) + return + } var sell = bot.balance.asset / 2 if (sell < bot.min_trade) { - get('console').log('[bot] HOLD') + get('console').log('[bot] HOLD', numeral(delta).format('0.0%')) return } bot.balance.asset -= sell - var price = close - (close * bot.markup) bot.balance.currency += sell * price var fee = (sell * price) * bot.fee bot.balance.currency -= fee - get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00')) + get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00'), numeral(delta).format('0.0%')) } printReport() } + lastClose = close }) function getNext () { var params = { @@ -127,7 +138,7 @@ module.exports = function container (get, set, clear) { sort: { time: 1 }, - limit: 100 + limit: bot.query_limit } get('db.ticks').select(params, function (err, ticks) { if (err) { @@ -141,6 +152,7 @@ module.exports = function container (get, set, clear) { if (!close) { initBalance.currency += initBalance.asset * tick.close initBalance.asset = 0 + lastClose = tick.close } close = tick.close tickStream.write(tick) @@ -154,12 +166,12 @@ module.exports = function container (get, set, clear) { time = time.replace(', ', ', 0') } var bar = getGraph() - get('console').log(bar + ' ' + numeral(close).format('$0,0.00'), time.grey, numeral(bot.balance.asset).format('00.000') + ' BTC/USD ' + numeral(bot.balance.currency).format('$,0.00')) + get('console').log(bar + ' ' + numeral(close).format('$0,0.00').yellow, time.grey, numeral(bot.balance.asset).format('00.000').white + ' BTC/USD '.grey + numeral(bot.balance.currency).format('$,0.00').yellow) setImmediate(getNext) }) } setImmediate(getNext) - get('console').log('mounted bot.') + get('console').log('mounted bot.', bot.sim ? 'SIMULATION' : 'REAL LIFE') cb && cb() } } \ No newline at end of file From 0cfaffc98414a1388080cb0d85a2bd4f69ccf86a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 10:16:14 -0700 Subject: [PATCH 0014/1659] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f059fa606c..f364ea4358 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # !ALPHA SOFTWARE. DO NOT USE FOR REAL TRADES. -zenbot is a passive trading bot for [GDAX](https://gdax.com/). zen trusts other bots. +zenbot is a passive trading bot for [GDAX](https://gdax.com/). zen trusts other bots and people. ## Usage From 2c57c96de3d2bc6353849198613152a302d3de0a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 10:21:40 -0700 Subject: [PATCH 0015/1659] use lastTick ms for query --- hooks/mountRecorder.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 58ef38b19d..6e63feb2ba 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -6,18 +6,20 @@ module.exports = function container (get, set, clear) { return function mountRecorder (cb) { var socket = get('utils.gdaxWebsocket') var counter = 0 + var lastTick = new Date().getTime() function onTick () { var trade_ticker = '' var params = { query: { time: { - $gt: new Date().getTime() - get('conf.tick_interval') + $gt: lastTick } }, sort: { time: 1 } } + lastTick = new Date().getTime() get('db.trades').select(params, function (err, trades) { if (err) return get('console').error('trade select err', err) var ticker = get('db.ticks').create(trades) From 22114556a356893785df0304ae6e2f27166f789f Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 10:33:34 -0700 Subject: [PATCH 0016/1659] sample conf --- conf/_codemap.js | 12 +++++++++--- gdax-config-sample.js | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 gdax-config-sample.js diff --git a/conf/_codemap.js b/conf/_codemap.js index b1c5796f4c..0ef9a76c15 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -14,8 +14,14 @@ module.exports = { 'tick_interval': 10000, 'tick_size': '10s', - 'gdax': require('../gdax-config.js'), - + 'gdax': function container (get, set) { + try { + return require('../gdax-config.js') + } + catch (e) { + return {} + } + }, 'bot': { balance: { asset: 0, @@ -29,4 +35,4 @@ module.exports = { query_limit: 100, crash_protection: 0.0019 } -} \ No newline at end of file +} diff --git a/gdax-config-sample.js b/gdax-config-sample.js new file mode 100644 index 0000000000..c4a41f0a92 --- /dev/null +++ b/gdax-config-sample.js @@ -0,0 +1,5 @@ +module.exports = { + passphrase: '', + key: '', + secret: '' +} From 51fc9a0b7b1973b6a0a8d7ca12d9a0b02e05f8ea Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 14:15:49 -0700 Subject: [PATCH 0017/1659] some base settings --- .gitignore | 3 +- conf/_codemap.js | 4 ++- hooks/mountBot.js | 63 ++++++++++++++++++++++++++++-------------- hooks/mountRecorder.js | 5 +++- 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index fda7f4ca7d..85216706f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules npm-debug.log db.json -gdax-config.js \ No newline at end of file +gdax-config.js +dump diff --git a/conf/_codemap.js b/conf/_codemap.js index 0ef9a76c15..cd308dbb1d 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -33,6 +33,8 @@ module.exports = { markup: 0.0002, fee: 0.0025, query_limit: 100, - crash_protection: 0.0019 + crash_protection: 0.02, + sell_for_less: 0.02, + buy_for_more: 0.05 } } diff --git a/hooks/mountBot.js b/hooks/mountBot.js index 2a0c3e9dca..4a2d9003cb 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -14,6 +14,8 @@ module.exports = function container (get, set, clear) { var runningVol = 0, runningTotal = 0 var high = 0, low = 10000, close = 0, vol = 0, lastClose = 0 var maxDiff = 0 + var buyPrice, sellPrice + var tradeVol = 0 function printReport () { var newBalance = JSON.parse(JSON.stringify(bot.balance)) @@ -23,7 +25,7 @@ module.exports = function container (get, set, clear) { if (diff > 0) diff = ('+' + numeral(diff).format('$0,0.00')).green if (diff === 0) diff = ('+' + numeral(diff).format('$0,0.00')).white if (diff < 0) diff = (numeral(diff).format('$0,0.00')).red - get('console').log('[bot]', diff) + get('console').log('[bot]', diff, numeral(tradeVol).format('0.000').white, 'BTC traded'.grey) } function getGraph () { @@ -77,56 +79,77 @@ module.exports = function container (get, set, clear) { vol += tick.vol } if (vol >= bot.min_vol) { + get('console').log(('[bot] volume trigger ' + side + ' ' + numeral(vol).format('0.0') + ' >= ' + numeral(bot.min_vol).format('0.0')).grey) vol = 0 // trigger if (side === 'BUY' && !bot.balance.currency) { - get('console').log('[bot] HOLD') - return + get('console').log('[bot] HOLD'.grey) + return finish() } else if (side === 'SELL' && !bot.balance.asset) { - get('console').log('[bot] HOLD') - return + get('console').log('[bot] HOLD'.grey) + return finish() } else if (side === 'BUY') { var delta = 1 - (lastClose / close) var price = close + (close * bot.markup) // add markup - if (delta >= bot.crash_protection) { - get('console').log('[bot] refusing to BUY at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.0%')) - return - } var spend = bot.balance.currency / 2 if (spend / price < bot.min_trade) { - get('console').log('[bot] HOLD', numeral(delta).format('0.0%')) - return + get('console').log(('[bot] HOLD ' + numeral(delta).format('0.000%')).grey) + return finish() + } + if (sellPrice && price > sellPrice) { + var sellDelta = 1 - (sellPrice / price) + if (sellDelta >= bot.buy_for_more) { + get('console').log(('[bot] refusing to BUY for more (sold for ' + numeral(sellPrice).format('$0,0.00') + ') at ' + numeral(price).format('$0,0.00') + ' ' + numeral(sellDelta).format('0.000%')).red) + return finish() + } } + if (delta >= bot.crash_protection) { + get('console').log(('[bot] refusing to BUY at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.000%')).red) + return finish() + } + buyPrice = price bot.balance.currency -= spend var size = spend / price + tradeVol += size bot.balance.asset += size var fee = (size * price) * bot.fee bot.balance.currency -= fee - get('console').log('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00'), numeral(delta).format('0.0%')) + get('console').log(('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00') + ' ' + numeral(delta).format('0.000%')).cyan) } else if (side === 'SELL') { var price = close - (close * bot.markup) // add markup var delta = 1 - (close / lastClose) - if (delta >= bot.crash_protection) { - get('console').log('[bot] refusing to SELL at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.0%')) - return - } var sell = bot.balance.asset / 2 if (sell < bot.min_trade) { - get('console').log('[bot] HOLD', numeral(delta).format('0.0%')) - return + get('console').log(('[bot] HOLD' + numeral(delta).format('0.000%')).grey) + return finish() + } + if (buyPrice && price < buyPrice) { + var buyDelta = 1 - (price / buyPrice) + if (buyDelta >= bot.sell_for_less) { + get('console').log(('[bot] refusing to SELL for less (bought for ' + numeral(buyPrice).format('$0,0.00') + ') at ' + numeral(price).format('$0,0.00') + ' ' + numeral(buyDelta).format('0.000%')).red) + return finish() + } } + if (delta >= bot.crash_protection) { + get('console').log(('[bot] refusing to SELL at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.000%')).red) + return finish() + } + sellPrice = price bot.balance.asset -= sell + tradeVol += sell bot.balance.currency += sell * price var fee = (sell * price) * bot.fee bot.balance.currency -= fee - get('console').log('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00'), numeral(delta).format('0.0%')) + get('console').log(('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00') + ' ' + numeral(delta).format('0.000%')).yellow) } printReport() } - lastClose = close + function finish () { + lastClose = close + } }) function getNext () { var params = { diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index 6e63feb2ba..f5c874cfd0 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -26,7 +26,10 @@ module.exports = function container (get, set, clear) { get('console').log('saw ' + counter + ' messages.' + ticker) if (counter === 0) { get('console').log('no messages in last tick. rebooting socket...') - socket.disconnect() + try { + socket.disconnect() + } + catch (e) {} clear('utils.gdaxWebsocket') clearInterval(interval) mountRecorder() From 113812c2f3776c7a08d38ec5eec6bebd5334e906 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 18:35:18 -0700 Subject: [PATCH 0018/1659] fix indexes and tweak algo --- bot.js | 1 + conf/_codemap.js | 12 ++++---- hooks/_codemap.js | 1 + hooks/mountBot.js | 66 +++++++++++++++++++++++++++++++----------- hooks/mountRecorder.js | 2 +- package.json | 3 +- 6 files changed, 61 insertions(+), 24 deletions(-) diff --git a/bot.js b/bot.js index 6fc0505f7c..d41ce508d0 100644 --- a/bot.js +++ b/bot.js @@ -8,6 +8,7 @@ try { require('motley-mongo') ], '@hooks.mount': [ + '#hooks.ensureIndexes', '#db.mountCollections', '#hooks.mountBot' ] diff --git a/conf/_codemap.js b/conf/_codemap.js index cd308dbb1d..1e0bc0a67b 100644 --- a/conf/_codemap.js +++ b/conf/_codemap.js @@ -25,16 +25,18 @@ module.exports = { 'bot': { balance: { asset: 0, - currency: 1000 + currency: 10000 }, - min_vol: 140, + min_vol: 150, + trade_amt: 0.70, min_trade: 0.01, sim: process.argv[2] !== '--real', markup: 0.0002, fee: 0.0025, query_limit: 100, - crash_protection: 0.02, - sell_for_less: 0.02, - buy_for_more: 0.05 + cooldown: 3, + crash_protection: 0.026, + sell_for_less: 0.05, + buy_for_more: 0.06 } } diff --git a/hooks/_codemap.js b/hooks/_codemap.js index 8e378cf009..e174594161 100644 --- a/hooks/_codemap.js +++ b/hooks/_codemap.js @@ -3,6 +3,7 @@ module.exports = { _ns: 'motley', _folder: 'hooks', + 'ensureIndexes': require('./ensureIndexes'), 'mountBackfiller': require('./mountBackfiller'), 'mountBot': require('./mountBot'), 'mountRecorder': require('./mountRecorder') diff --git a/hooks/mountBot.js b/hooks/mountBot.js index 4a2d9003cb..9f2c3ec22f 100644 --- a/hooks/mountBot.js +++ b/hooks/mountBot.js @@ -2,6 +2,7 @@ var numeral = require('numeral') , colors = require('colors') , tb = require('timebucket') , through = require('through') + , zerofill = require('zero-fill') module.exports = function container (get, set, clear) { return function mountBot (cb) { @@ -16,17 +17,7 @@ module.exports = function container (get, set, clear) { var maxDiff = 0 var buyPrice, sellPrice var tradeVol = 0 - - function printReport () { - var newBalance = JSON.parse(JSON.stringify(bot.balance)) - newBalance.currency += newBalance.asset * close - newBalance.asset = 0 - var diff = newBalance.currency - initBalance.currency - if (diff > 0) diff = ('+' + numeral(diff).format('$0,0.00')).green - if (diff === 0) diff = ('+' + numeral(diff).format('$0,0.00')).white - if (diff < 0) diff = (numeral(diff).format('$0,0.00')).red - get('console').log('[bot]', diff, numeral(tradeVol).format('0.000').white, 'BTC traded'.grey) - } + var cooldown = 0 function getGraph () { runningTotal += ((high + low + close) / 3) * periodVol @@ -91,9 +82,24 @@ module.exports = function container (get, set, clear) { return finish() } else if (side === 'BUY') { + if (cooldown > 0) { + get('console').log(('[bot] HOLD too soon to BUY').grey) + return finish() + } + cooldown = bot.cooldown var delta = 1 - (lastClose / close) var price = close + (close * bot.markup) // add markup - var spend = bot.balance.currency / 2 + var vwap = runningTotal / runningVol + var vwapDiff = price - vwap + var spend + if (vwapDiff > 0) { + // buy more when price is rising + spend = bot.balance.currency * bot.trade_amt + } + else { + // buy less when price is falling + spend = bot.balance.currency * (1 - bot.trade_amt) + } if (spend / price < bot.min_trade) { get('console').log(('[bot] HOLD ' + numeral(delta).format('0.000%')).grey) return finish() @@ -107,6 +113,7 @@ module.exports = function container (get, set, clear) { } if (delta >= bot.crash_protection) { get('console').log(('[bot] refusing to BUY at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.000%')).red) + cooldown = 0 return finish() } buyPrice = price @@ -119,9 +126,24 @@ module.exports = function container (get, set, clear) { get('console').log(('[bot] BUY ' + numeral(size).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00') + ' ' + numeral(delta).format('0.000%')).cyan) } else if (side === 'SELL') { + if (cooldown > 0) { + get('console').log(('[bot] HOLD too soon to SELL').grey) + return finish() + } + cooldown = bot.cooldown var price = close - (close * bot.markup) // add markup var delta = 1 - (close / lastClose) - var sell = bot.balance.asset / 2 + var vwap = runningTotal / runningVol + var vwapDiff = price - vwap + var sell + if (vwapDiff < 0) { + // sell more when price is falling + sell = bot.balance.asset * bot.trade_amt + } + else { + // sell less when price is rising + sell = bot.balance.asset * (1 - bot.trade_amt) / 2 + } if (sell < bot.min_trade) { get('console').log(('[bot] HOLD' + numeral(delta).format('0.000%')).grey) return finish() @@ -135,17 +157,17 @@ module.exports = function container (get, set, clear) { } if (delta >= bot.crash_protection) { get('console').log(('[bot] refusing to SELL at ' + numeral(price).format('$0,0.00') + ': crash protection ' + numeral(delta).format('0.000%')).red) + cooldown = 0 return finish() - } + } sellPrice = price bot.balance.asset -= sell tradeVol += sell bot.balance.currency += sell * price var fee = (sell * price) * bot.fee bot.balance.currency -= fee - get('console').log(('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00') + ' ' + numeral(delta).format('0.000%')).yellow) + get('console').log(('[bot] SELL ' + numeral(sell).format('00.000') + ' BTC at ' + numeral(price).format('$0,0.00') + ' ' + numeral(delta).format('0.000%')).cyan) } - printReport() } function finish () { lastClose = close @@ -171,6 +193,7 @@ module.exports = function container (get, set, clear) { if (!ticks.length) { return setTimeout(getNext, get('conf.tick_interval')) } + var periodVol = 0 ticks.forEach(function (tick) { if (!close) { initBalance.currency += initBalance.asset * tick.close @@ -181,7 +204,9 @@ module.exports = function container (get, set, clear) { tickStream.write(tick) minTime = tick.time counter++ + periodVol += tick.vol }) + if (cooldown) cooldown-- var date = new Date(minTime) var tzMatch = date.toString().match(/\((.*)\)/) var time = date.toLocaleString() + ' ' + tzMatch[1] @@ -189,7 +214,14 @@ module.exports = function container (get, set, clear) { time = time.replace(', ', ', 0') } var bar = getGraph() - get('console').log(bar + ' ' + numeral(close).format('$0,0.00').yellow, time.grey, numeral(bot.balance.asset).format('00.000').white + ' BTC/USD '.grey + numeral(bot.balance.currency).format('$,0.00').yellow) + var newBalance = JSON.parse(JSON.stringify(bot.balance)) + newBalance.currency += newBalance.asset * close + newBalance.asset = 0 + var diff = newBalance.currency - initBalance.currency + if (diff > 0) diff = ('+' + numeral(diff).format('$0,0.00')).green + if (diff === 0) diff = ('+' + numeral(diff).format('$0,0.00')).white + if (diff < 0) diff = (numeral(diff).format('$0,0.00')).red + get('console').log(bar + ' ' + numeral(close).format('$0,0.00').yellow, zerofill(4, numeral(periodVol).format('0'), ' ').white, time.grey, numeral(bot.balance.asset).format('00.000').white + ' BTC/USD '.grey + numeral(bot.balance.currency).format('$,0.00').yellow + ' ' + diff) setImmediate(getNext) }) } diff --git a/hooks/mountRecorder.js b/hooks/mountRecorder.js index f5c874cfd0..3987492d66 100644 --- a/hooks/mountRecorder.js +++ b/hooks/mountRecorder.js @@ -12,7 +12,7 @@ module.exports = function container (get, set, clear) { var params = { query: { time: { - $gt: lastTick + $gte: lastTick } }, sort: { diff --git a/package.json b/package.json index be1ccecda3..9125a5175c 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "motley-mongo": "^2.0.5", "numeral": "^1.5.3", "through": "^2.3.8", - "timebucket": "^0.3.4" + "timebucket": "^0.3.4", + "zero-fill": "^2.2.3" } } From d4651e3acba7caa803c78d8745676fd94ff2e0b2 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 28 Jun 2016 20:46:50 -0700 Subject: [PATCH 0019/1659] switch to bin format --- _codemap.js | 1 + bin/zenbot | 69 +++++++++++++++++ bot/_codemap.js | 8 ++ hooks/mountBot.js => bot/brain.js | 122 +++++++++++++----------------- bot/simulator.js | 50 ++++++++++++ bot/zen.js | 5 ++ conf/_codemap.js | 6 +- hooks/_codemap.js | 1 - package.json | 2 + 9 files changed, 192 insertions(+), 72 deletions(-) create mode 100755 bin/zenbot create mode 100644 bot/_codemap.js rename hooks/mountBot.js => bot/brain.js (69%) create mode 100644 bot/simulator.js create mode 100644 bot/zen.js diff --git a/_codemap.js b/_codemap.js index 50352abd44..b00547cd23 100644 --- a/_codemap.js +++ b/_codemap.js @@ -1,6 +1,7 @@ module.exports = { _ns: 'motley', _maps: [ + require('./bot/_codemap'), require('./conf/_codemap'), require('./db/_codemap'), require('./hooks/_codemap'), diff --git a/bin/zenbot b/bin/zenbot new file mode 100755 index 0000000000..94139b02a6 --- /dev/null +++ b/bin/zenbot @@ -0,0 +1,69 @@ +#!/usr/bin/env node +var motley = require('motley') + +var program = require('commander') + .version(require('../package.json').version) + +program + .command('sim') + .description('run the simulator') + .option('--start