features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Generates features from Order Book data\n",
    "Data format:\n",
    "    - data_dir folder contains CSVs with order book data for a given stock\n",
    "    (msft, goog, amzn, aapl) at a given depth (1-10)\n",
    "    - Order book data is second-by-second\n",
    "'''\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "'''\n",
    "Raw Data Format:\n",
    "    columns (n=1:10):\n",
    "        - datetime (YYYY-MM-DD H:M:S)\n",
    "        - bid{n}\n",
    "        - ask{n}\n",
    "        - bsize{n}\n",
    "        - asize{n}\n",
    "        - bnum{n}\n",
    "        - anum{n}\n",
    "        - vwap\n",
    "        - notional\n",
    "        - volume\n",
    "        - last_price\n",
    "        - mid\n",
    "        - spread\n",
    "        - wmid\n",
    "        - last_size\n",
    "        - last_SRO\n",
    "'''\n",
    "\n",
    "data_dir = '../../ProjectData/'\n",
    "\n",
    "\n",
    "def mergeOrderBookDays(data_dir, out_path, prefixes):\n",
    "    '''\n",
    "    Raw data is separated into files by date (e.g. msft-20170417)\n",
    "    Merge files with the same prefix (ticker) by day\n",
    "    '''\n",
    "    files = os.listdir(data_dir)\n",
    "\n",
    "    for prefix in prefixes:\n",
    "        data = pd.DataFrame()\n",
    "        fnames = [f for f in files if (str(f).startswith(prefix) and str(f).endswith('orderbook.csv'))]\n",
    "        print(fnames)\n",
    "        for f in fnames:\n",
    "            print(data_dir + str(f))\n",
    "            data = data.append(pd.read_csv(data_dir + str(f)))\n",
    "        data.to_csv(out_path, index = False)\n",
    "    #return data\n",
    "\n",
    "def createResponseVariable(data, response_type = 'Classification'):\n",
    "    '''\n",
    "    Generates response variable for raw input data.\n",
    "    Response variable will be:\n",
    "        - mid price of next tick order book (response_type = 'Regression')\n",
    "        - Up , Down, No Change (response_type = 'Classification')\n",
    "    '''\n",
    "    if response_type.upper() == 'REGRESSION':\n",
    "        response_col = [data.loc[i+1, 'direct.mid'] for i in range(len(data)-1)]\n",
    "\n",
    "    elif response_type.upper() == 'CLASSIFICATION':\n",
    "        response_col = []\n",
    "        for i in range(len(data)-1):\n",
    "            current_price = data.loc[i, 'direct.mid']\n",
    "            next_price = data.loc[i+1, 'direct.mid']\n",
    "            diff = next_price - current_price\n",
    "            if diff == 0:\n",
    "                response_col.append(0)\n",
    "            elif diff > 0:\n",
    "                response_col.append(1)\n",
    "            elif diff < 0:\n",
    "                response_col.append(2)\n",
    "\n",
    "    data = data[:-1] # get rid of last row, which won't have a response variable\n",
    "    data['Response'] = response_col\n",
    "    return data\n",
    "\n",
    "def calculateImbalance(data):\n",
    "    '''\n",
    "    Calulate Order Book imbalance\n",
    "    '''\n",
    "    pass\n",
    "\n",
    "def calculateMeanPricesAndVolumes(data):\n",
    "    '''\n",
    "    Mean Bid/Ask, Prices/Volumes\n",
    "    sum(Price_i)/n\n",
    "    '''\n",
    "    data['meanBid'] = 'NA'\n",
    "    bid_col_list = ['direct.bid{}'.format(i) for i in range(1,11)]\n",
    "    print(bid_col_list)\n",
    "    #for i in range(len(data)):\n",
    "    data['meanBid'] = data[bid_col_list].sum(axis=1)\n",
    "\n",
    "    data['meanAsk'] = 'NA'\n",
    "    ask_col_list = ['direct.ask{}'.format(i) for i in range(1,11)]\n",
    "    #for i in range(len(data)):\n",
    "    data['meanAsk'] = data[ask_col_list].sum(axis=1)\n",
    "\n",
    "    data['meanBidNum'] = 'NA'\n",
    "    bidNum_col_list = ['direct.bnum{}'.format(i) for i in range(1,11)]\n",
    "    #for i in range(len(data)):\n",
    "    data['meanBidNum'] = data[bidNum_col_list].sum(axis=1)\n",
    "\n",
    "    data['meanAskNum'] = 'NA'\n",
    "    askNum_col_list = ['direct.anum{}'.format(i) for i in range(1,11)]\n",
    "    #for i in range(len(data)):\n",
    "    data['meanAskNum'] = data[askNum_col_list].sum(axis=1)\n",
    "\n",
    "    var_cols = bid_col_list + ask_col_list + bidNum_col_list + askNum_col_list\n",
    "    var_cols += ['meanBid', 'meanAsk', 'meanBidNum', 'meanAskNum']\n",
    "    return data, var_cols\n",
    "\n",
    "def calculateSpreadsAndMidPrices(data):\n",
    "    '''\n",
    "    bid-ask spreads and mid-prices\n",
    "\n",
    "    P_ask - P_bid{i=1,...,n}\n",
    "    P_ask + P_bid{i=1,...,n}\n",
    "    '''\n",
    "    # calculate spreads\n",
    "    for i in range(1,11):\n",
    "        #data['spread_{}'.format(i)] = 'NA'\n",
    "        #bid = data.loc[i, 'direct.bid{}'.format(i)]\n",
    "        #ask = data.loc[j, 'direct.ask{}'.format(i)]\n",
    "        spread = [data.loc[j, 'direct.ask{}'.format(i)] - data.loc[j, 'direct.bid{}'.format(i)] for j in range(len(data))]\n",
    "        data['spread_{}'.format(i)] = spread\n",
    "\n",
    "    # calculate mid prices\n",
    "    for i in range(1,11):\n",
    "        #data['midPrice_{}'.format(i)] = 'NA'\n",
    "        #bid = data.loc[j, 'direct.bid{}'.format(i)]\n",
    "        #ask = data.loc[j, 'direct.ask{}'.format(i)]\n",
    "        midPrice = [(data.loc[j, 'direct.ask{}'.format(i)] + data.loc[j, 'direct.bid{}'.format(i)])/2 for j in range(len(data))]\n",
    "        data['midPrice_{}'.format(i)] = midPrice\n",
    "\n",
    "    var_cols_spread = ['spread_{}'.format(i) for i in range(1,11)]\n",
    "    var_cols_mp = ['midPrice_{}'.format(i) for i in range(1,11)]\n",
    "    var_cols = var_cols_spread + var_cols_mp\n",
    "    return data, var_cols\n",
    "\n",
    "def createFeatures(data_path, out_path, response_type):\n",
    "    '''\n",
    "    Generates features from Order Book Data\n",
    "\n",
    "    Inputs:\n",
    "        - data_path: path to order book data\n",
    "        - out_path: path to place generated file\n",
    "    Output:\n",
    "        - featureMatrix: data frame containing original data and features\n",
    "    '''\n",
    "\n",
    "    data = pd.read_csv(data_path)\n",
    "    data, meanPriceVol_vars = calculateMeanPricesAndVolumes(data)\n",
    "\n",
    "    data, spreadMidPrice_vars = calculateSpreadsAndMidPrices(data)\n",
    "    #data = calculateImbalance(data)\n",
    "    data = createResponseVariable(data, response_type)\n",
    "\n",
    "    feature_vars = meanPriceVol_vars + spreadMidPrice_vars + ['Response']\n",
    "    data = data[feature_vars]\n",
    "    data.to_csv(out_path, index = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "createFeatures(data_path, out_path, response_type)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}