sample_notebooks/BaseLine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy\n",
    "import pickle as pkl\n",
    "import os\n",
    "os.chdir(os.path.join(os.getcwd(), \"..\"))\n",
    "import finviz as fz\n",
    "import yfinance as yf\n",
    "import calendar\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import precision_recall_curve, auc, average_precision_score, log_loss\n",
    "from sklearn.metrics import PrecisionRecallDisplay\n",
    "\n",
    "from src.utils import data_layer, get_dataset, hist_multi_plot \n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "perc_space = 0.8\n",
    "perc_time = 0.8\n",
    "\n",
    "WINDOW_SIZE = 21 # Include window + target\n",
    "BATCH_SIZE = 8\n",
    "\n",
    "data_path = os.path.join(os.getcwd(), \"data\")\n",
    "sp_100_file = os.path.join(data_path, \"sp-100-index-07-02-2020.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and Data formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_year(current_month, current_year):\n",
    "    dict_cont = {}\n",
    "    dict_cont[\"current_year\"] = current_year\n",
    "    dict_cont[\"previous_month\"] = current_month\n",
    "    def _infer_year(x):\n",
    "        delta_months = dict_cont[\"previous_month\"] - x\n",
    "        if x>=1 and delta_months>=0:\n",
    "            dict_cont[\"previous_month\"] = x\n",
    "        else:\n",
    "            dict_cont[\"previous_month\"] = x\n",
    "            dict_cont[\"current_year\"]-=1\n",
    "        return dict_cont[\"current_year\"]\n",
    "    return _infer_year\n",
    "\n",
    "def data_layer(sp_100_file, data_path, perc_space, perc_time):\n",
    "    sp_100_df = pd.read_csv(sp_100_file)\n",
    "    simbols = sp_100_df.Symbol.values[:-1]\n",
    "    map_month2month_number = dict((v,k) for k,v in enumerate(calendar.month_abbr))\n",
    "    stock = {}\n",
    "    cont = 0\n",
    "    avg_amount = pd.DataFrame()\n",
    "    ordered_columns = ['Open', 'High', 'Low', 'Close', 'Volume', 'Perc_sp100', 'volatility', \n",
    "                       'change_price_beg_end', 'change_price', 'Volume_rel_price', 'Vol_rel_volatility',\n",
    "                       'Dividends', 'Stock Splits', 'Buy', 'Sale', 'Option Exercise']\n",
    "    for simbol in simbols:\n",
    "        try:\n",
    "            insider_info = fz.get_insider(simbol)\n",
    "            insider_data = pd.DataFrame.from_dict(insider_info)\n",
    "            insider_data[[\"month_name\", \"day\"]]= insider_data.Date.str.split(\" \", expand=True)\n",
    "            insider_data[\"month\"] = insider_data[\"month_name\"].map(map_month2month_number)\n",
    "            insider_data[\"year\"] = insider_data[\"month\"].apply(infer_year(datetime.datetime.now().month,\n",
    "                                                                        datetime.datetime.now().year))\n",
    "            insider_data[\"date\"] = pd.to_datetime(insider_data[[\"year\", \"month\", \"day\"]], format=\"%y%m%d\") \n",
    "            cont = cont + 1\n",
    "        except Exception as inst:\n",
    "            print(simbol, end=\" \")\n",
    "            print(type(inst), end=\" \")# the exception instance\n",
    "            print(inst) \n",
    "            continue\n",
    "        curr_ticker = yf.Ticker(simbol)\n",
    "        curr_hist = curr_ticker.history(period=\"1y\")\n",
    "        curr_table = pd.pivot_table(insider_data,\n",
    "                                    index=['date'],\n",
    "                                    columns=['Transaction'], aggfunc={\"Transaction\": len})\n",
    "        curr_hist[\"volatility\"] = (curr_hist.High - curr_hist.Low)/(curr_hist.Open + np.finfo(float).eps)\n",
    "        curr_hist[\"change_price_beg_end\"] = curr_hist.Open - curr_hist.Close\n",
    "        curr_hist[\"change_price\"] = curr_hist.High - curr_hist.Low\n",
    "        curr_hist[\"Volume_rel_price\"] = curr_hist.Volume*curr_hist[\"change_price\"]\n",
    "        curr_hist[\"Vol_rel_volatility\"] = curr_hist.Volume*curr_hist[\"volatility\"]\n",
    "        curr_table = ~curr_table.isnull()\n",
    "        curr_table.columns = [col[1] for col in curr_table.columns]\n",
    "        curr_hist = curr_hist.merge(curr_table, how=\"left\", left_index=True, right_index=True)\n",
    "        curr_hist = curr_hist.dropna(subset=['Open', 'High', 'Low', 'Close', 'Volume'])\n",
    "        curr_hist = curr_hist.fillna(False)\n",
    "        if \"Buy\" not in curr_hist.columns:\n",
    "            curr_hist[\"Buy\"] = False\n",
    "        if \"Sale\" not in curr_hist.columns:\n",
    "            curr_hist[\"Sale\"] = False\n",
    "        if \"Option Exercise\" not in curr_hist.columns:\n",
    "            curr_hist[\"Option Exercise\"] = False\n",
    "        stock[simbol] = curr_hist\n",
    "        stock[simbol][\"Amount_negotiated\"] = stock[simbol][\"Volume\"]*stock[simbol][\"Close\"]\n",
    "        avg_amount[simbol] = stock[simbol][\"Amount_negotiated\"]\n",
    "        \n",
    "    avg_amount_mean = avg_amount.mean(axis=1)\n",
    "    for key, val in stock.items():\n",
    "        stock[key][\"Perc_sp100\"] = stock[key][\"Volume\"]*stock[key][\"Close\"]/avg_amount_mean\n",
    "        stock[key] = stock[key][ordered_columns]\n",
    "    return stock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BKNG <class 'IndexError'> list index out of range\n",
      "BRK.B <class 'requests.exceptions.HTTPError'> 404 Client Error: Not Found for url: https://finviz.com/quote.ashx?t=BRK.B\n",
      "C <class 'IndexError'> list index out of range\n",
      "DD <class 'IndexError'> list index out of range\n",
      "EXC <class 'IndexError'> list index out of range\n",
      "GOOGL <class 'IndexError'> list index out of range\n",
      "GS <class 'IndexError'> list index out of range\n"
     ]
    }
   ],
   "source": [
    "stock = data_layer(sp_100_file, data_path, perc_space, perc_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_train_space = int(len(stock.keys())*perc_space)\n",
    "n_train_out_space = len(stock.keys())-n_train_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_space = np.random.choice(list(stock.keys()), size=n_train_space, replace=False).tolist()\n",
    "test_space = list(set(stock.keys()) - set(train_space))\n",
    "samples = stock[\"AAPL\"].shape[0]\n",
    "train_samples = int((samples-WINDOW_SIZE)*perc_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data loader object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Companies:  ['GD', 'JNJ', 'QCOM', 'NFLX', 'COP', 'AIG', 'MMM', 'NKE', 'SO', 'DHR', 'GE', 'KMI', 'OXY', 'BMY', 'USB', 'COF', 'KO', 'MS', 'TGT', 'GOOG', 'PYPL', 'BA', 'AMT', 'UPS', 'BK', 'SPG', 'ORCL', 'NEE', 'F', 'FB', 'DUK', 'GM', 'ABT', 'WMT', 'DIS', 'LOW', 'TXN', 'MRK', 'UNP', 'IBM', 'CVS', 'PEP', 'KHC', 'AMGN', 'VZ', 'MA', 'BIIB', 'LMT', 'HON', 'T', 'ACN', 'LLY', 'CVX', 'ADBE', 'EMR', 'AMZN', 'GILD', 'PFE', 'PM', 'CRM', 'V', 'CSCO', 'CL', 'SBUX', 'MCD', 'MDT', 'WBA', 'CAT', 'INTC', 'AXP', 'BAC', 'CMCSA', 'ABBV']\n"
     ]
    }
   ],
   "source": [
    "train_ds, val_ds = get_dataset(stock, train_space, train_samples=train_samples, window_size=WINDOW_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Companies:  ['MSFT', 'WFC', 'MDLZ', 'COST', 'AAPL', 'NVDA', 'CHTR', 'BLK', 'SLB', 'DOW', 'PG', 'UNH', 'MET', 'FDX', 'ALL', 'TMO', 'HD', 'XOM', 'JPM']\n"
     ]
    }
   ],
   "source": [
    "test_ds = get_dataset(stock, test_space, train_samples=None, window_size=WINDOW_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = train_ds.shuffle(buffer_size=8000)\n",
    "train_ds = train_ds.batch(BATCH_SIZE)\n",
    "train_ds = train_ds.prefetch(16)\n",
    "val_ds = val_ds.batch(BATCH_SIZE)\n",
    "test_ds = test_ds.batch(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((<tf.Tensor: shape=(8, 20, 16), dtype=float64, numpy=\n",
       "  array([[[ 54.68,  55.47,  53.99, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 55.12,  55.92,  54.72, ...,   0.  ,   1.  ,   1.  ],\n",
       "          [ 55.82,  56.18,  55.58, ...,   0.  ,   0.  ,   0.  ],\n",
       "          ...,\n",
       "          [ 56.41,  56.68,  56.07, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 56.52,  57.99,  56.48, ...,   0.  ,   0.  ,   1.  ],\n",
       "          [ 58.18,  58.82,  57.91, ...,   0.  ,   1.  ,   1.  ]],\n",
       "  \n",
       "         [[116.51, 117.57, 115.97, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [116.49, 117.54, 115.62, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [115.92, 116.65, 114.18, ...,   0.  ,   0.  ,   0.  ],\n",
       "          ...,\n",
       "          [112.88, 114.89, 112.62, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [112.94, 114.29, 112.94, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [114.55, 115.63, 114.41, ...,   0.  ,   0.  ,   0.  ]],\n",
       "  \n",
       "         [[ 59.35,  59.4 ,  58.8 , ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 59.21,  59.58,  59.04, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 59.66,  60.47,  59.64, ...,   0.  ,   1.  ,   1.  ],\n",
       "          ...,\n",
       "          [ 59.13,  59.3 ,  58.71, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 58.78,  59.35,  58.62, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 59.18,  59.41,  58.84, ...,   0.  ,   0.  ,   0.  ]],\n",
       "  \n",
       "         ...,\n",
       "  \n",
       "         [[ 85.63,  85.78,  82.85, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 86.66,  89.19,  86.17, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 89.66,  91.54,  88.17, ...,   0.  ,   0.  ,   0.  ],\n",
       "          ...,\n",
       "          [ 92.79,  93.07,  90.68, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 91.78,  94.17,  91.45, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [ 94.47,  94.57,  92.54, ...,   0.  ,   0.  ,   0.  ]],\n",
       "  \n",
       "         [[113.81, 115.68, 112.15, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [111.66, 115.02, 111.31, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [112.68, 118.59, 111.85, ...,   0.  ,   0.  ,   0.  ],\n",
       "          ...,\n",
       "          [128.78, 129.32, 128.  , ...,   0.  ,   0.  ,   0.  ],\n",
       "          [129.43, 129.74, 127.61, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [127.72, 129.67, 127.14, ...,   0.  ,   0.  ,   0.  ]],\n",
       "  \n",
       "         [[145.61, 146.46, 143.83, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [143.69, 145.62, 141.62, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [144.39, 145.93, 143.14, ...,   0.  ,   0.  ,   0.  ],\n",
       "          ...,\n",
       "          [139.99, 141.64, 139.53, ...,   0.  ,   0.  ,   0.  ],\n",
       "          [141.78, 142.48, 141.  , ...,   0.  ,   0.  ,   0.  ],\n",
       "          [141.32, 142.41, 141.03, ...,   0.  ,   0.  ,   0.  ]]])>,\n",
       "  <tf.Tensor: shape=(8, 13), dtype=float64, numpy=\n",
       "  array([[ 5.79100000e+01,  5.80900000e+01,  5.73100000e+01,\n",
       "           5.77000000e+01,  6.24230000e+06,  4.61185476e-01,\n",
       "           1.34691763e-02,  2.10000000e-01,  7.80000000e-01,\n",
       "           4.86899400e+06,  8.40786393e+04,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 1.15160000e+02,  1.15360000e+02,  1.13790000e+02,\n",
       "           1.14100000e+02,  2.72060000e+06,  3.45630933e-01,\n",
       "           1.36332060e-02,  1.06000000e+00,  1.57000000e+00,\n",
       "           4.27134200e+06,  3.70905002e+04,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 5.90300000e+01,  5.94300000e+01,  5.89200000e+01,\n",
       "           5.92900000e+01,  4.17870000e+06,  2.75857572e-01,\n",
       "           8.63967474e-03, -2.60000000e-01,  5.10000000e-01,\n",
       "           2.13113700e+06,  3.61026088e+04,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 9.42000000e+01,  9.72600000e+01,  9.42000000e+01,\n",
       "           9.69700000e+01,  1.39365000e+07,  1.14533399e+00,\n",
       "           3.24840764e-02, -2.77000000e+00,  3.06000000e+00,\n",
       "           4.26456900e+07,  4.52714331e+05,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 1.16370000e+02,  1.17050000e+02,  1.15750000e+02,\n",
       "           1.16620000e+02,  6.44550000e+06,  8.42455636e-01,\n",
       "           1.11712641e-02, -2.50000000e-01,  1.30000000e+00,\n",
       "           8.37915000e+06,  7.20043826e+04,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 9.28600000e+01,  9.61200000e+01,  9.27800000e+01,\n",
       "           9.57000000e+01,  7.23680000e+06,  6.92303696e-01,\n",
       "           3.59681241e-02, -2.84000000e+00,  3.34000000e+00,\n",
       "           2.41709120e+07,  2.60294120e+05,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 1.25020000e+02,  1.25210000e+02,  1.22010000e+02,\n",
       "           1.23060000e+02,  1.48707000e+07,  1.26456580e+00,\n",
       "           2.55959047e-02,  1.96000000e+00,  3.20000000e+00,\n",
       "           4.75862400e+07,  3.80629019e+05,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 1.43040000e+02,  1.43520000e+02,  1.42410000e+02,\n",
       "           1.42990000e+02,  1.04810000e+06,  2.51923311e-01,\n",
       "           7.76006711e-03,  5.00000000e-02,  1.11000000e+00,\n",
       "           1.16339100e+06,  8.13332634e+03,  0.00000000e+00,\n",
       "           0.00000000e+00]])>),\n",
       " <tf.Tensor: shape=(8, 1, 3), dtype=float64, numpy=\n",
       " array([[[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]],\n",
       " \n",
       "        [[0., 0., 0.]]])>)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = iter(train_ds)\n",
    "next(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    inputs = tf.keras.Input(shape=(20,16))\n",
    "    x = tf.keras.layers.BatchNormalization(axis=2)(inputs)\n",
    "    x_0 = tf.keras.layers.GRU(64, return_sequences=False)(x)\n",
    "    x_1 = tf.keras.layers.GRU(64, return_sequences=False)(x)\n",
    "    x = tf.keras.layers.Concatenate()([x_0, x_1])\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    sigmoids = tf.keras.layers.Dense(3, activation=\"sigmoid\")(x)\n",
    "    outputs = tf.keras.backend.expand_dims(sigmoids, axis=1)\n",
    "    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 20, 16)]     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 20, 16)       64          input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       (None, 64)           15744       batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (None, 64)           15744       batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 128)          0           gru[0][0]                        \n",
      "                                                                 gru_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 128)          512         concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 3)            387         batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_ExpandDims (TensorF [(None, 1, 3)]       0           dense[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 32,451\n",
      "Trainable params: 32,163\n",
      "Non-trainable params: 288\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"binary_crossentropy\",\n",
    "    optimizer=\"adam\",\n",
    "    metrics=[\"accuracy\", \"AUC\", tf.keras.metrics.AUC(curve=\"PR\", name=\"PR_AUC\")])\n",
    "es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',#val_loss\n",
    "                                               patience=2,\n",
    "                                               mode='min',\n",
    "                                               restore_best_weights=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "WARNING:tensorflow:From /Users/camilovelasquez/anaconda3/envs/insider/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "1689/1689 [==============================] - 26s 16ms/step - loss: 0.1594 - accuracy: 0.0980 - auc: 0.6716 - PR_AUC: 0.0626 - val_loss: 0.0927 - val_accuracy: 0.0365 - val_auc: 0.8252 - val_PR_AUC: 0.3502\n",
      "Epoch 2/20\n",
      "1689/1689 [==============================] - 37s 22ms/step - loss: 0.1197 - accuracy: 0.0692 - auc: 0.7413 - PR_AUC: 0.1473 - val_loss: 0.1008 - val_accuracy: 0.0451 - val_auc: 0.7623 - val_PR_AUC: 0.3402\n",
      "Epoch 3/20\n",
      "1689/1689 [==============================] - 31s 18ms/step - loss: 0.1182 - accuracy: 0.0772 - auc: 0.7625 - PR_AUC: 0.1484 - val_loss: 0.0955 - val_accuracy: 0.0622 - val_auc: 0.8397 - val_PR_AUC: 0.2597\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_ds, \n",
    "                    validation_data=val_ds,\n",
    "                    epochs=20, \n",
    "                    callbacks = [es_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "554/554 [==============================] - 3s 5ms/step - loss: 0.1132 - accuracy: 0.0535 - auc: 0.7612 - PR_AUC: 0.1256\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.11315745860338211,\n",
       " 0.05353512614965439,\n",
       " 0.7612084150314331,\n",
       " 0.12563152611255646]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing predictions and looking if it is learning anything\n",
    "dataframe start from the 20, example df[20:] are all the samples in test, and each company has ~232 records, so to iterate would be accooording to 232 and addding 20 to the origiinal df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = []\n",
    "for sample in test_ds:\n",
    "    y_test.append(sample[1].numpy())\n",
    "y_test = np.squeeze(np.concatenate(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Company:  MSFT\n",
      "Company:  WFC\n",
      "Company:  MDLZ\n",
      "Company:  COST\n",
      "Company:  AAPL\n",
      "Company:  NVDA\n",
      "Company:  CHTR\n",
      "Company:  BLK\n",
      "Company:  SLB\n",
      "Company:  DOW\n",
      "Company:  PG\n",
      "Company:  UNH\n",
      "Company:  MET\n",
      "Company:  FDX\n",
      "Company:  ALL\n",
      "Company:  TMO\n",
      "Company:  HD\n",
      "Company:  XOM\n",
      "Company:  JPM\n"
     ]
    }
   ],
   "source": [
    "out = model.predict(test_ds)\n",
    "out = np.squeeze(out)\n",
    "aux_stock = {}\n",
    "cont = 0\n",
    "for idx, company in enumerate(test_space):\n",
    "    print(\"Company: \", company)\n",
    "    curr_hist = stock[company]\n",
    "    curr_index = curr_hist.index[WINDOW_SIZE-1:]\n",
    "    curr_size = len(curr_index)\n",
    "    pred_df = pd.DataFrame(out[cont:cont+curr_size],\n",
    "                           index=curr_hist.index[WINDOW_SIZE-1:],\n",
    "                           columns=[\"Pred_Buy\", \"Pred_Option\", \"Pred_Sale\"])\n",
    "    aux_stock[company] = stock[company].merge(pred_df, how=\"left\", \n",
    "                                              left_index=True, right_index=True)\n",
    "    cont = cont + curr_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.concatenate([out, y_test], axis=1), \n",
    "             columns=[\"pred_Buy\", \"pred_Option\", \"pred_Sale\", \"Buy\", \"Option\", \"Sale\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "means = df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Precision:  0.010835968561982715\n",
      "Strawman Crossentropy 0.041720669601262764\n",
      "Binary Crossentropy 0.04407000022082522\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASjklEQVR4nO3dfZBd9X3f8ffHCHUrmwJ6MCFaYonIdis705iRHTLukBRosJVEMAlDoTGIIpdCqUuczgQSWj/QpBPPtEndMZNEE4JFEiNh1ak0BJGEp3HtCdgi4BhEbck8WCtkEAqithNAkr/94x5gESvtlfbuww+9XzM7ex5+55zvb+/qo3N/95yzqSokSe1503QXIEk6Mga4JDXKAJekRhngktQoA1ySGmWAS1KjDHAdlZI8keTs6a5DmggDXDqIJJ9IsjfJ97qvR5P84nTXJb3MAFfzksyaxN2vq6q3VNVbgF8G/jjJSZN4PKlvBrhmrG6Y49eSbEnyXJKbkgwl+ekkI0muSfId4KYkb0pybZJvJdmd5NYkc0ft6+IkT3brrjuSeqrqz4HvAj/a7fPSJF86oOZKsiTJe5M8neSYUet+IcnXjuiHIY3BANdM90vAOfRC8x3Af+qW/xAwF3gbcDnwEeA84KeAHwaeA24ASLIU+F3g4m7dPGD4cIpIz88Cs4Et47Wvqq8Cu4GfGbX4YuDmwzmudCgGuGa6z1TV9qr6W+A3gYu65T8APl5VL1bV3wNXANdV1UhVvQh8Aji/G145H7itqr7YrfvP3fb9uCDJHuB7wEbgv1bVnj63XQN8CKB7N3AO8Lk+t5XGNZljh9IgbB81/SS9M2iAXVX1wqh1bwP+NMnoYN4PnNRt88p+qur7SXb3efxbq+rlEF4E3Jbk+ar6/T62/WPg0SRvBi4A/k9V7ezzuNK4PAPXTHfKqOkfAZ7qpg98jOZ24INVdcKor6Gq2gHsHL2fJHPoDaMclqp6AtgE/Hy36PvAnFH7/aED2u8A/gr4BXrDJ390uMeUDsUA10x3VZLhbgjiOmDdQdr9HvCbSd4GkGRBknO7deuBn0vyz5LMBq7nCH73kwwDHwAe6RZ9DXhXkh9PMkRv2OZANwO/CvwY8IXDPaZ0KAa4ZrrPAX8BPAZ8C/iNg7T7NL0x6r9I8l3gPuAnAKrqEeCqbl876X3AOdLn8f/ly9eBA18Fvgx8stvvN+n9Z3AnsBX40hjb/ynd8E5V/V2fx5T6Ev+gg2aqJE8AH66qO6e7lolI8i3g37beD808noFLk6i7c7OAu6e7Fr3xGOA6qiXZNOpW+dFfvz6Afd9L7/rzq6qq38sWpb45hCJJjfIMXJIaNaU38syfP78WLVo0lYeUpOY98MADz1bVggOXT2mAL1q0iM2bN0/lISWpeUmeHGu5QyiS1CgDXJIaZYBLUqN8GqGkpu3du5eRkRFeeOGF8RvPcENDQwwPD3Psscf21d4Al9S0kZERjjvuOBYtWkSS6S7niFUVu3fvZmRkhMWLF/e1jUMokpr2wgsvMG/evKbDGyAJ8+bNO6x3Ega4pOa1Ht4vO9x+GOCS1CjHwCW9ody55emB7u/spSeN2+aOO+7g6quvZv/+/Xz4wx/m2muvfc36F198kUsuuYQHHniAefPmsW7dOgZxV3ozZ+B3bnn6lS9Jmin279/PVVddxaZNm9iyZQu33HILW7ZseU2bG2+8kRNPPJFt27bx0Y9+lGuuuWYgx24mwCVpJvrKV77CkiVLOPXUU5k9ezYXXnghGzZseE2bDRs2sHLlSgDOP/987rrrLgbxJFgDXJImYMeOHZxyyqt/e3t4eJgdO3YctM2sWbM4/vjj2b1794SPbYBLUqMMcEmagIULF7J9+/ZX5kdGRli4cOFB2+zbt4/nn3+eefPmTfjYBrgkTcB73/tetm7dyuOPP85LL73E2rVrWbFixWvarFixgjVr1gCwfv16zjzzzIFcu+5lhJLeUPq57G+QZs2axWc+8xnOOecc9u/fz2WXXca73vUuPvaxj7Fs2TJWrFjBqlWruPjii1myZAlz585l7dq1gzn2QPYiSUex5cuXs3z58tcsu/7661+ZHhoa4vOf//zAj+sQiiQ1qpkz8PlP3f3qzNKLpq8QSZohPAOXpEYZ4JLUKANckhplgEtSo5r5EFOS+vKNTYPd3zs/OG6Tyy67jNtuu423vvWtPPzww69bX1VcffXV3H777cyZM4fPfvaznHbaaRMura8z8CQfTfJIkoeT3JJkKMniJPcn2ZZkXZLZE65Gkhp06aWXcscddxx0/aZNm9i6dStbt25l9erVXHnllQM57rgBnmQh8B+AZVX1buAY4ELgU8DvVNUS4Dlg1UAqkqTGnHHGGcydO/eg6zds2MAll1xCEk4//XT27NnDzp07J3zcfsfAZwH/MMksYA6wEzgTWN+tXwOcN+FqJOkNqJ9Hzh6JcQO8qnYA/w34Nr3gfh54ANhTVfu6ZiPAwrG2T3J5ks1JNu/atWvCBUuSevoZQjkROBdYDPww8GbgA/0eoKpWV9Wyqlq2YMGCIy5UklrVzyNnj0Q/QyhnA49X1a6q2gt8AXg/cEI3pAIwDEz8/YAkvQGtWLGCm2++marivvvu4/jjj+fkk0+e8H77uYzw28DpSeYAfw+cBWwG7gHOB9YCK4ENB92DJE2VPi77G7SLLrqIe++9l2effZbh4WE++clPsnfvXgCuuOIKli9fzu23386SJUuYM2cON91000COO26AV9X9SdYDfw3sAx4EVgN/BqxN8hvdshsHUpEkNeaWW2455Pok3HDDDQM/bl838lTVx4GPH7D4MeB9A69IktQXb6WXpEYZ4JKaV1XTXcJAHG4/DHBJTRsaGmL37t3Nh3hVsXv3boaGhvrexodZSWra8PAwIyMjvBFuFBwaGmJ4eLjv9ga4pKYde+yxLF68eLrLmBYOoUhSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqP6CvAkJyRZn+T/Jnk0yU8mmZvkL5Ns7b6fONnFSpJe1e8Z+KeBO6rqHwP/FHgUuBa4q6reDtzVzUuSpsi4AZ7keOAM4EaAqnqpqvYA5wJrumZrgPMmq0hJ0uv1cwa+GNgF3JTkwSR/kOTNwElVtbNr8x3gpLE2TnJ5ks1JNu/atWswVUuS+grwWcBpwO9W1XuA73PAcElVFVBjbVxVq6tqWVUtW7BgwUTrlSR1+gnwEWCkqu7v5tfTC/Snk5wM0H1/ZnJKlCSNZdwAr6rvANuTvLNbdBawBdgIrOyWrQQ2TEqFkqQxzeqz3UeAP0kyG3gM+Nf0wv/WJKuAJ4ELJqdESdJY+grwqnoIWDbGqrMGW44kqV/eiSlJjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGtV3gCc5JsmDSW7r5hcnuT/JtiTrksyevDIlSQc6nDPwq4FHR81/CvidqloCPAesGmRhkqRD6yvAkwwDPwv8QTcf4ExgfddkDXDeZBQoSRpbv2fg/wP4VeAH3fw8YE9V7evmR4CFA65NknQI4wZ4kp8DnqmqB47kAEkuT7I5yeZdu3YdyS4kSWPo5wz8/cCKJE8Aa+kNnXwaOCHJrK7NMLBjrI2ranVVLauqZQsWLBhAyZIk6CPAq+rXqmq4qhYBFwJ3V9UvAfcA53fNVgIbJq1KSdLrTOQ68GuAX0myjd6Y+I2DKUmS1I9Z4zd5VVXdC9zbTT8GvG/wJUmS+uGdmJLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDVq3ABPckqSe5JsSfJIkqu75XOT/GWSrd33Eye/XEnSy/o5A98H/MeqWgqcDlyVZClwLXBXVb0duKublyRNkXEDvKp2VtVfd9PfBR4FFgLnAmu6ZmuA8yarSEnS6x3WGHiSRcB7gPuBk6pqZ7fqO8BJB9nm8iSbk2zetWvXBEqVJI3Wd4AneQvwv4Bfrqr/N3pdVRVQY21XVaurallVLVuwYMGEipUkvaqvAE9yLL3w/pOq+kK3+OkkJ3frTwaemZwSJUlj6ecqlAA3Ao9W1W+PWrURWNlNrwQ2DL48SdLBzOqjzfuBi4GvJ3moW/brwG8BtyZZBTwJXDA5JUqSxjJugFfVl4AcZPVZgy1HktQv78SUpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktSofv4m5sz2jU2vTr/zg9NXhyRNMc/AJalRTZ6B37nl6Vemzz5mGguRpGnkGbgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGtXkw6zmP3X3qzOnnHB4G49+/Cz4CFpJzfIMXJIaZYBLUqMMcElqVJNj4KM9tH3PK9M//s5pLESSptiEzsCTfCDJN5JsS3LtoIqSJI3viM/AkxwD3AD8C2AE+GqSjVW1ZVDFHa7X/Km1pSdNVxmSNCUmMoTyPmBbVT0GkGQtcC4wbQE+2ugwH23+U3teM++wi6RWTSTAFwLbR82PAD9xYKMklwOXd7PfS/KNIzzefODZI9z2EP7V4Hc5WJPU7xntaOwzHJ39ts/9edtYCyf9Q8yqWg2snuh+kmyuqmUDKKkpR2O/j8Y+w9HZb/s8MRP5EHMHcMqo+eFumSRpCkwkwL8KvD3J4iSzgQuBjYMpS5I0niMeQqmqfUn+PfDnwDHAH1bVIwOr7PUmPAzTqKOx30djn+Ho7Ld9noBU1aD2JUmaQt5KL0mNMsAlqVEzLsDHuz0/yT9Isq5bf3+SRVNf5WD10edfSbIlyd8kuSvJmNeEtqbfRzEk+cUklaT5y8366XOSC7rX+5Ekn5vqGidDH7/jP5LkniQPdr/ny6ejzkFK8odJnkny8EHWJ8n/7H4mf5PktMM+SFXNmC96H4Z+CzgVmA18DVh6QJt/B/xeN30hsG66656CPv9zYE43fWXrfe63312744AvAvcBy6a77il4rd8OPAic2M2/dbrrnqJ+rwau7KaXAk9Md90D6PcZwGnAwwdZvxzYBAQ4Hbj/cI8x087AX7k9v6peAl6+PX+0c4E13fR64KwkmcIaB23cPlfVPVX1d93sffSuuW9dP681wH8BPgW8MJXFTZJ++vxvgBuq6jmAqnpmimucDP30u4B/1E0fDzw1hfVNiqr6IvC3h2hyLnBz9dwHnJDk5MM5xkwL8LFuz194sDZVtQ94Hpg3JdVNjn76PNoqev9rt27cfndvKU+pqj+bysImUT+v9TuAdyT5cpL7knxgyqqbPP30+xPAh5KMALcDH5ma0qbV4f7bf53mnwd+NEnyIWAZ8FPTXctkS/Im4LeBS6e5lKk2i94wyk/Te6f1xSQ/VlV7DrlV+y4CPltV/z3JTwJ/lOTdVfWD6S5sJptpZ+D93J7/Spsks+i93do9JdVNjr4eSZDkbOA6YEVVvThFtU2m8fp9HPBu4N4kT9AbI9zY+AeZ/bzWI8DGqtpbVY8D36QX6C3rp9+rgFsBquqvgCF6D316I5vw40hmWoD3c3v+RmBlN30+cHd1nwg0atw+J3kP8Pv0wvuNMCYK4/S7qp6vqvlVtaiqFtEb+19RVZunp9yB6Of3+3/TO/smyXx6QyqPTWWRk6Cffn8bOAsgyT+hF+C7prTKqbcRuKS7GuV04Pmq2nlYe5juT2oP8snsN+l9an1dt+x6ev94offCfh7YBnwFOHW6a56CPt8JPA081H1tnO6ap6LfB7S9l8avQunztQ69oaMtwNeBC6e75inq91Lgy/SuUHkI+JnprnkAfb4F2AnspffOahVwBXDFqNf6hu5n8vUj+f32VnpJatRMG0KRJPXJAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmN+v+1Egr7dUgA2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWL0lEQVR4nO3dfbRddX3n8feHhPAgISgJDiXhwRofAEXxjvjQVkCgSFuY+lToMBaHJa0j2lUdZ+jUooJrWnXBWuNadAAHitqqVdvaWGMZBdRZFpxcBJGEQSMGk2jL5SkIAZKbfOePcy4cbu7DCbn7XG72+7XWXffs3/6dvb+/e+69n7P3PnvvVBWSpPbaY7YLkCTNLoNAklrOIJCkljMIJKnlDAJJarn5s13Azlq8eHEdfvjhs12GJM0pN998871VtWSieXMuCA4//HCGh4dnuwxJmlOS3D3ZPHcNSVLLGQSS1HIGgSS1nEEgSS1nEEhSyzUWBEmuTnJPktsnmZ8kn0iyNsltSY5tqhZJ0uSa3CK4Bjh1ivlvAJZ3v84D/meDtUiSJtFYEFTVt4H7p+hyBvDp6rgJOCDJwU3Vs2rd/Vz6v+9ky+j2plYhSXPSbB4jOARY3zO9odu2gyTnJRlOMjwyMvK0Vva9ux/gE9evZXS7QSBJvebEweKqurKqhqpqaMmSCc+QliQ9TbMZBBuBZT3TS7ttkqQBms0gWAG8rfvpoVcBm6rq57NYjyS1UmMXnUvyOeB4YHGSDcAHgT0BqupyYCVwGrAW2Ay8valaJEmTaywIquqsaeYX8K6m1i9J6s+cOFgsSWqOQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyzUaBElOTXJnkrVJLphg/qFJbkhyS5LbkpzWZD2SpB01FgRJ5gGXAW8AjgTOSnLkuG4fAL5QVS8HzgT+oql6JEkTa3KL4JXA2qq6q6q2AJ8HzhjXp4D9u48XAT9rsB5J0gSaDIJDgPU90xu6bb0+BJydZAOwEnj3RAtKcl6S4STDIyMjTdQqSa012weLzwKuqaqlwGnAZ5LsUFNVXVlVQ1U1tGTJkoEXKUm7syaDYCOwrGd6abet17nAFwCq6kZgb2BxgzVJksZpMghWAcuTHJFkAZ2DwSvG9fkp8HqAJC+mEwTu+5GkAWosCKpqFDgfuBa4g86ng1YnuSjJ6d1u7wPekeT7wOeAc6qqmqpJkrSj+U0uvKpW0jkI3Nt2Yc/jNcBrm6xBkjS12T5YLEmaZQaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1XKNBkOTUJHcmWZvkgkn6vDXJmiSrk3y2yXokSTua39SCk8wDLgNOBjYAq5KsqKo1PX2WA38MvLaqHkhyUFP1SJIm1uQWwSuBtVV1V1VtAT4PnDGuzzuAy6rqAYCquqfBeiRJE2gyCA4B1vdMb+i29XoB8IIk30lyU5JTJ1pQkvOSDCcZHhkZaahcSWqnvoIgyWuTfD3JD5PcleQnSe6agfXPB5YDxwNnAZ9McsD4TlV1ZVUNVdXQkiVLZmC1kqQx/R4juAr4I+BmYFufz9kILOuZXtpt67UB+G5VbQV+kuSHdIJhVZ/rkCTton53DW2qqq9V1T1Vdd/Y1zTPWQUsT3JEkgXAmcCKcX2+TGdrgCSL6ewqmoktDUlSn/rdIrghyceBvwMeH2usqu9N9oSqGk1yPnAtMA+4uqpWJ7kIGK6qFd15pyRZQ2dL4/19BIwkaQb1GwTHdb8P9bQVcOJUT6qqlcDKcW0X9jwu4L3dL0nSLOgrCKrqhKYLkSTNjn4/NbQoyaVjH+FMckmSRU0XJ0lqXr8Hi68GfgG8tfv1EPCXTRUlSRqcfo8R/HJVvaln+sNJbm2iIEnSYPW7RfBokl8Zm0jyWuDRZkqSJA1Sv1sE7wQ+1T0uEOB+4JymipIkDU6/nxq6FTgmyf7d6YcarUqSNDBTBkGSs6vqr5K8d1w7AFV1aYO1SZIGYLotgmd1vy9suhBJ0uyYMgiq6oru9w8PphxJ0qD1e0LZx5Lsn2TPJNclGUlydtPFSZKa1+/HR0/pHiD+TWAd8Hzg/U0VJUkanH6DYGwX0m8AX6yqTQ3VI0kasH7PI/jHJP+Pzklk70yyBHisubIkSYPS1xZBVV0AvAYY6t5N7BF2vBG9JGkOmu48ghOr6vokb+xp6+3yd00VJkkajOl2Db0OuB74rQnmFQaBJM15051H8MHu97cPphxJ0qD1ex7Bf09yQM/0s5N8pLmyJEmD0u/HR99QVQ+OTVTVA8BpzZQkSRqkfoNgXpK9xiaS7APsNUV/SdIc0e95BH8NXJdk7PaUbwc+1UxJkqRB6vd+BB9N8n3gpG7TxVV1bXNlSZIGpd8tAoA7gNGq+kaSfZMsrKpfNFWYJGkw+v3U0DuALwFXdJsOAb7cVFGSpMHp92Dxu4DXAg8BVNWPgIOaKkqSNDj9BsHjVbVlbCLJfDpnFkuS5rh+g+BbSf4bsE+Sk4EvAl9prixJ0qD0GwT/FRgBfgD8PrAS+EBTRUmSBmfaTw0lmQesrqoXAZ9sviRJ0iBNu0VQVduAO5McOoB6JEkD1u+uoWcDq7s3rl8x9jXdk5KcmuTOJGuTXDBFvzclqSRD/RYuSZoZ/Z5Q9qc7u+DuLqXLgJOBDcCqJCuqas24fguBPwS+u7PrkCTtuunuULY38AfA8+kcKL6qqkb7XPYrgbVVdVd3WZ+nc3vLNeP6XQx8FHj/TtQtSZoh0+0a+hQwRCcE3gBcshPLPgRY3zO9odv2hCTHAsuq6qtTLSjJeUmGkwyPjIzsRAmSpOlMt2voyKp6CUCSq4D/O1MrTrIHcClwznR9q+pK4EqAoaEhT2STpBk03RbB1rEHO7FLaMxGYFnP9NJu25iFwNHAN5OsA14FrPCAsSQN1nRbBMckeaj7OHTOLH6o+7iqav8pnrsKWJ7kCDoBcCbwu2Mzq2oTsHhsOsk3gf9cVcM7PQpJ0tM23c3r5z3dBVfVaJLzgWuBecDVVbU6yUXAcFVN+/FTSVLzduZ+BDutqlbSuRxFb9uFk/Q9vslaJEkT6/eEMknSbsogkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCklms0CJKcmuTOJGuTXDDB/PcmWZPktiTXJTmsyXokSTtqLAiSzAMuA94AHAmcleTIcd1uAYaq6qXAl4CPNVWPJGliTW4RvBJYW1V3VdUW4PPAGb0dquqGqtrcnbwJWNpgPZKkCTQZBIcA63umN3TbJnMu8LWJZiQ5L8lwkuGRkZEZLFGS9Iw4WJzkbGAI+PhE86vqyqoaqqqhJUuWDLY4SdrNzW9w2RuBZT3TS7ttT5HkJOBPgNdV1eMN1iNJmkCTWwSrgOVJjkiyADgTWNHbIcnLgSuA06vqngZrkSRNorEgqKpR4HzgWuAO4AtVtTrJRUlO73b7OLAf8MUktyZZMcniJEkNaXLXEFW1Elg5ru3CnscnNbl+SdL0nhEHiyVJs8cgkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWo5g0CSWs4gkKSWmz/bBQzKtioAbl3/IPsumHrYhz5nX57zrAWDKEuSZl1rguALq9YD8Luf/O60fY/6pf356nt+dUbWu2rd/ay/f/O0/fbecx4nvfi5LJjvRpqkwWpNEPzisVEALn3rMTx738nf7V/x7R/zgw2buPfhx6dc3uOj2xndtp2QSfsUxVsuv7HvGv/ynH/LCS86qO/+kjQTWhMEY37tBUtYvN9ek85/+zWrABj6yDdmbJ0L957PP777Vyadv/aehzn3U8M8Prp9xtYpSf1qXRD06+Izjppy/p/+w2oALnnLMVP2+/SN6/izN76Uww581qR9Hti8FYD3fO4WTjnquVMu78BnLeDC3zqKeXtMviUiSTuj0SBIcirwP4B5wP+qqj8fN38v4NPAK4D7gN+pqnVN1tSv//Dqw6ecf/m37uKoX9qfN71i6ZT9ppsPcF93N9SWbdtZ8/OHJu1318gjAHzmprunDJaf3Nvpt//eU7+8D3V3l53wwiVT9rvhzhEADl6096R9Nm/ZxqZHt065nF4fe/NLOeSAfSad/4vHRrl94yYOO3DfKZfzo3se5u77HuHIgxdN2e87P76XRfvsyZEH7z9lv3+4dSMvP/TZTxlrTdCvJmiscT3vf3gLf3/LRs5+1WE9z+v0qur0H1vOWBu9bT19CvjGHf/K41u3c8pRz32ibWx59PYd97ze5fY+57Gt29h7z3lTvg4AD2zewsuWHcBe8+dN2e+xrdt48bifb8a9Xxm/K3X8/E6fqRumW8ZEb5GSnXvOjnU92fD46Dbuf2TLE3sWnnwNe15Men/2T8576utdvd17nvdk33rqInnRv1nIsudM/TfxdDQWBEnmAZcBJwMbgFVJVlTVmp5u5wIPVNXzk5wJfBT4naZq6sflZx/Lmp9N/s94zHcuOHHG1vn6Fz+X33/d83jfyS+c8mDxX910Nx/48u0cd8SBLFk4+e6tsSB447FTh9A1/7wOgPse2dJXnb+6fPGk87Zth7/93gaOO+I5HPe8Ayft95kb1/HA5q38ly/d1tc6+3Xt6n/tq9/X10zfb919m9lz3rh/HBP9e5mmaWxX3zX/vI79957/xD+jpNMvSfd7z1qemPfkesfaHuxuOd66/sGnPJ8Jljf2PMa3d+eNbi/u+PlDHLDvnty+cdOkP4t7H+78bqz8wb9M2keD85F/d/RT3ljMlNREb21mYsHJq4EPVdWvd6f/GKCq/qynz7XdPjcmmQ/8C7CkpihqaGiohoeHd7qeV1z8de57ZAs3f+AkDpziGIGatX178aGvrOY3XnLwDu/Sej26dRu/eGwrxyw9YMrljW4vlizci333nPrd6uj2Yt4ecZfa0/Dw46Nsmeb41ej27fz0vs1PeSc9/q94/B/1hFtV4xqne874rbCJNt92dhnT1X3PQ49x78NbOGbpoieSf+yNwpMB3v2eTNA2tqSJ5mWHvr3LPnjR3k/7/1eSm6tqaKJ5Te4aOgRY3zO9AThusj5VNZpkE3AgcG9vpyTnAecBHHrooU+rmEveegzv/uwthsAs22OPcNEZRw98vQsMgKdtv73mQx9/NgctnHzXoZ7Z5sSH1qvqyqoaqqqhJUum3p89meNfeBA/+PCvz3BlkjT3NRkEG4FlPdNLu20T9unuGlpE56CxJGlAmgyCVcDyJEckWQCcCawY12cF8Hvdx28Grp/q+IAkaeY1doygu8//fOBaOh8fvbqqVie5CBiuqhXAVcBnkqwF7qcTFpKkAWr0PIKqWgmsHNd2Yc/jx4C3NFmDJGlqc+JgsSSpOQaBJLWcQSBJLWcQSFLLNXaJiaYkGQHufppPX8y4s5ZbwDG3g2Nuh10Z82FVNeEZuXMuCHZFkuHJrrWxu3LM7eCY26GpMbtrSJJaziCQpJZrWxBcOdsFzALH3A6OuR0aGXOrjhFIknbUti0CSdI4BoEktdxuGQRJTk1yZ5K1SS6YYP5eSf6mO/+7SQ4ffJUzq48xvzfJmiS3Jbkuyczf+HTAphtzT783Jakkc/6jhv2MOclbu6/16iSfHXSNM62P3+1Dk9yQ5Jbu7/dps1HnTElydZJ7ktw+yfwk+UT353FbkmN3eaVVtVt90bnk9Y+B5wELgO8DR47r85+Ay7uPzwT+ZrbrHsCYTwD27T5+ZxvG3O23EPg2cBMwNNt1D+B1Xg7cAjy7O33QbNc9gDFfCbyz+/hIYN1s172LY/414Fjg9knmnwZ8jc6tjV8FfHdX17k7bhG8ElhbVXdV1Rbg88AZ4/qcAXyq+/hLwOsz1Z3Un/mmHXNV3VBVm7uTN9G5Y9xc1s/rDHAx8FHgsUEW15B+xvwO4LKqegCgqu4ZcI0zrZ8xF7B/9/Ei4GcDrG/GVdW36dyfZTJnAJ+ujpuAA5IcvCvr3B2D4BBgfc/0hm7bhH2qahTYBBw4kOqa0c+Ye51L5x3FXDbtmLubzMuq6quDLKxB/bzOLwBekOQ7SW5KcurAqmtGP2P+EHB2kg107n/y7sGUNmt29u99Wo3emEbPPEnOBoaA1812LU1KsgdwKXDOLJcyaPPp7B46ns5W37eTvKSqHpzVqpp1FnBNVV2S5NV07np4dFVtn+3C5ordcYtgI7CsZ3ppt23CPknm09mcvG8g1TWjnzGT5CTgT4DTq+rxAdXWlOnGvBA4GvhmknV09qWumOMHjPt5nTcAK6pqa1X9BPghnWCYq/oZ87nAFwCq6kZgbzoXZ9td9fX3vjN2xyBYBSxPckSSBXQOBq8Y12cF8Hvdx28Grq/uUZg5atoxJ3k5cAWdEJjr+41hmjFX1aaqWlxVh1fV4XSOi5xeVcOzU+6M6Od3+8t0tgZIspjOrqK7BlnkDOtnzD8FXg+Q5MV0gmBkoFUO1grgbd1PD70K2FRVP9+VBe52u4aqajTJ+cC1dD5xcHVVrU5yETBcVSuAq+hsPq6lc1DmzNmreNf1OeaPA/sBX+weF/9pVZ0+a0Xvoj7HvFvpc8zXAqckWQNsA95fVXN2a7fPMb8P+GSSP6Jz4PicufzGLsnn6IT54u5xjw8CewJU1eV0joOcBqwFNgNv3+V1zuGflyRpBuyOu4YkSTvBIJCkljMIJKnlDAJJajmDQJJaziCQJpBkW5Jbk9ye5CtJDpjh5a/rfs6fJA/P5LKlnWUQSBN7tKpeVlVH0znX5F2zXZDUFINAmt6NdC/qleSXk/xTkpuT/J8kL+q2PzfJ3yf5fvfrNd32L3f7rk5y3iyOQZrUbndmsTSTksyjc/mCq7pNVwJ/UFU/SnIc8BfAicAngG9V1W93n7Nft/9/rKr7k+wDrEryt3P5TF/tngwCaWL7JLmVzpbAHcDXk+wHvIYnL9MBsFf3+4nA2wCqahudS5sDvCfJb3cfL6NzATiDQM8oBoE0sUer6mVJ9qVznZt3AdcAD1bVy/pZQJLjgZOAV1fV5iTfpHNBNOkZxWME0hS6d3V7D50Lm20GfpLkLfDEvWOP6Xa9js4tQEkyL8kiOpc3f6AbAi+icyls6RnHIJCmUVW3ALfRuQHKvwfOTfJ9YDVP3jbxD4ETkvwAuJnOvXP/CZif5A7gz+lcClt6xvHqo5LUcm4RSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktdz/B1B0abBk9s/oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_col = \"pred_Buy\"\n",
    "col = \"Buy\"\n",
    "hist_multi_plot(df, pred_col, col, bins=np.arange(0,1, 0.01))\n",
    "precision, recall, _ = precision_recall_curve(df[col], df[pred_col])\n",
    "pr_display = PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "average_precision = average_precision_score(df[col], df[pred_col])\n",
    "print(\"Average Precision: \", average_precision)\n",
    "strawman_ce = log_loss(df[col], np.ones(df[col].shape)*means[col])\n",
    "print(\"Strawman Crossentropy\", strawman_ce)\n",
    "bce = log_loss(df[col], df[pred_col])\n",
    "print(\"Binary Crossentropy\", bce)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Precision:  0.06152576250758002\n",
      "Strawman Crossentropy 0.10366509320611689\n",
      "Binary Crossentropy 0.1014334942695975\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUlElEQVR4nO3dfZBldX3n8fdHZsiEyALzIDvLEAdEWTApkWqNRsv4gPJQW4OpsC5sAsMCO4FFi5hsFUQSlWzcMlXR1KakdCfFw5CNPITozhTLsALCUlqimVEEHMoM8hB6GGEYhEAMyIzf/ePegabppm/3vd13fjPvV9WtPvec3znn+5vu+fTp33m4qSokSe15zbALkCTNjAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1x7pCQPJTluSPten2TlMPatvYsBrr1akqOTrEvydJJnktyW5Nensf6nk/yvsfOq6sSqWjP4aqWXM8C120syb5a2+wbgm8A9wGHAvwG+CnwtyTtnY5/SIBngGpruMMcfJtmU5CdJrkiyIMl7k4wmuTDJj4ErkrwmyUVJfpRke5Lrkiwcs63TkzzcXXZxjyV8GvhWVV1cVU9W1TNV9ZfAXwN/1t3u8iSVZFWSR5NsTfJfu8tOAD4B/Ickzyb5fnf+7UnO6U6/JskfdWt7PMlVSQ4Yt+2VSf4xyRPTqF0ywDV0vw0cD7wBeBPwR935/xpYCLweWAV8DPgw8Bt0jpR/AlwKnWEQ4IvA6d1li4BlPez7g8DfTjD/OuBdSX5xzLz3AW8EPgRcmOS4qroJ+O/AtVX12qp6ywTbOrP7eh9wOPBa4Avj2rwbOBL4APDJJEf1ULtkgGvovlBVj1TVk8BngNO6838OfKqqnq+qfwHOBS6uqtGqep7O0fMp3eGVU4AbquqO7rI/7q4/lcXA1gnmb6Xzf2PhmHmXVNU/V9U9wBVj6pzKbwOfr6oHqupZ4A+BU8cNC11SVf9SVd8Hvg9M9ItAeoVZGVuUpuGRMdMP0zmCBthWVc+NWfZ64KtJxgbzTuDg7jovbqeq/jnJ9h72/QSwdIL5S+n8AvgJ8LpJ6vzVHrZPt7aHx607r1v3Lj8eM/1TOkfp0pQ8AtewHTpm+peBR7vT4x+T+QhwYlUdOOa1oKq20DlifnE7SfajM4wylVuAfz/B/I/QGRv/6QzqHO9ROr98xq67A3ish/qkV2WAa9jOT7Kse0LyYuDaSdp9CfhMktcDJFmS5OTusuuBf5fk3Un2Bf6E3n62LwF+PclnkixMsn+SjwFnABeOa/vHSfZL8mbgP42p8zFgeZLJ9nc18PEkhyV5LS+Nme/ooT7pVRngGrYvA18DHgB+BPzpJO3+B7COziV+zwB3Ar8GUFU/AM7vbmsrnaGP0al2XFWb6ZxAfAvwUHfd3wKOr6pvjmv+/4D7gVuBP6+qr3Xn7zoJuj3JdyfYzeV0rmq5A3gQeI7OCVmpb/EDHTQsSR4CzqmqW4Zdy2SSLKcTvPM9atbuxiNwSWqUAa49Wve5JM9O8PrEsGuT+uUQiiQ1yiNwSWrUnN7Is3jx4lq+fPlc7lKSmrdx48YnqmrJ+PlzGuDLly9nw4YNc7lLSWpekocnmu8QiiQ1ygCXpEYZ4JLUKJ9GKKlpL7zwAqOjozz33HNTN97NLViwgGXLljF//vye2hvgkpo2OjrK/vvvz/Lly0ky7HJmrKrYvn07o6OjHHbYYT2t4xCKpKY999xzLFq0qOnwBkjCokWLpvWXhAEuqXmth/cu0+2HAS5JjXIMXNIe5ZZNg/2wo+OOPnjqRsBNN93EBRdcwM6dOznnnHO46KKLXrb8+eef54wzzmDjxo0sWrSIa6+9ln7vTG/yCPyWTY+9+JKkYdu5cyfnn38+69evZ9OmTVx99dVs2rTpZW0uu+wyDjroIO6//34+/vGPc+GF4z/0afqaDHBJ2p185zvf4YgjjuDwww9n33335dRTT2Xt2rUva7N27VpWrlwJwCmnnMKtt95Kv0+DNcAlqU9btmzh0ENf+tzrZcuWsWXLlknbzJs3jwMOOIDt27f3tV8DXJIaZYBLUp8OOeQQHnnkkRffj46Ocsghh0zaZseOHTz99NMsWrSor/0a4JLUp7e97W1s3ryZBx98kJ/97Gdcc801rFix4mVtVqxYwZo1awC4/vrref/739/39etTXkaYZAFwB/AL3fbXV9WnkhwGXAMsAjYCp1fVz/qqRpL61Otlf4M0b948vvCFL3D88cezc+dOzjrrLN785jfzyU9+kpGREVasWMHZZ5/N6aefzhFHHMHChQu55ppr+t7vlJ+Jmc6viF+qqmeTzAe+AVwA/D7wlaq6JsmXgO9X1RdfbVsjIyM1iA90GHv54DC+WZJ2H/fddx9HHXXUsMsYmIn6k2RjVY2MbzvlEEp1PNt9O7/7KuD9wPXd+WuAD/dT9Ex5TbikvVVPY+BJ9klyF/A4cDPwI+CpqtrRbTIKHDLJuquSbEiyYdu2bYOoWZJEjwFeVTur6hhgGfB24N/2uoOqWl1VI1U1smTJKz6TU5I0Q9O6CqWqngJuA94JHJhk10nQZcCWSVeUJA3clAGeZEmSA7vTvwh8ELiPTpCf0m22Elg78RYkSbOhl6cRLgXWJNmHTuBfV1U3JNkEXJPkT4HvAZfNYp2SpHGmDPCquht46wTzH6AzHi5Ju48frh/s9o48ccomZ511FjfccAOve93ruPfee1+xvKq44IILuPHGG9lvv/248sorOfbYY/suzTsxJalPZ555JjfddNOky9evX8/mzZvZvHkzq1ev5rzzzhvIfg1wSerTe97zHhYuXDjp8rVr13LGGWeQhHe84x089dRTbN26te/9GuCSNMt6edzsTBjgktQoA1ySZlkvj5udCQNckmbZihUruOqqq6gq7rzzTg444ACWLl3a93b9VHpJe5YeLvsbtNNOO43bb7+dJ554gmXLlnHJJZfwwgsvAHDuuedy0kknceONN3LEEUew3377ccUVVwxkvwa4JPXp6quvftXlSbj00ksHvl+HUCSpUQa4JDXKAJfUvKk+WawV0+2HAS6paQsWLGD79u3Nh3hVsX37dhYsWNDzOp7ElNS0ZcuWMTo6yp7wiV8LFixg2bJlPbc3wCU1bf78+Rx22GHDLmMoHEKRpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRzV8HvvjRr7/05ujThleIJM0xj8AlqVEGuCQ1asoAT3JoktuSbErygyQXdOd/OsmWJHd1XyfNfrmSpF16GQPfAfxBVX03yf7AxiQ3d5f9RVX9+eyVJ0mazJQBXlVbga3d6WeS3Af0/3HKkqS+TGsMPMly4K3At7uzPprk7iSXJzloknVWJdmQZMOe8LhHSdpd9BzgSV4L/B3we1X1T8AXgTcAx9A5Qv/cROtV1eqqGqmqkSVLlgygZEkS9BjgSebTCe+/qaqvAFTVY1W1s6p+DvwV8PbZK1OSNF4vV6EEuAy4r6o+P2b+0jHNfhO4d/DlSZIm08tVKO8CTgfuSXJXd94ngNOSHAMU8BDwu7NSoSRpQr1chfINIBMsunHw5UiSeuWdmJLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGtXLZ2I245ZNj704fdzRBw+xEkmafR6BS1KjDHBJapQBLkmNMsAlqVEGuCQ1asoAT3JoktuSbErygyQXdOcvTHJzks3drwfNfrmSpF16uYxwB/AHVfXdJPsDG5PcDJwJ3FpVn01yEXARcOHslfqSxY9+fS52I0m7tSmPwKtqa1V9tzv9DHAfcAhwMrCm22wN8OHZKlKS9ErTGgNPshx4K/Bt4OCq2tpd9GNgwjtnkqxKsiHJhm3btvVRqiRprJ4DPMlrgb8Dfq+q/mnssqoqoCZar6pWV9VIVY0sWbKkr2IlSS/pKcCTzKcT3n9TVV/pzn4sydLu8qXA47NToiRpIr1chRLgMuC+qvr8mEXrgJXd6ZXA2sGXJ0maTC9XobwLOB24J8ld3XmfAD4LXJfkbOBh4COzU6IkaSJTBnhVfQPIJIs/MNhyJEm98k5MSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUqHnDLmCQFj/69ZfeHH3a8AqRpDngEbgkNWrKAE9yeZLHk9w7Zt6nk2xJclf3ddLslilJGq+XI/ArgRMmmP8XVXVM93XjYMuSJE1lygCvqjuAJ+egFknSNPQzBv7RJHd3h1gOmqxRklVJNiTZsG3btj52J0kaa6YB/kXgDcAxwFbgc5M1rKrVVTVSVSNLliyZ4e4kSePN6DLCqnps13SSvwJuGFhFg/LD9S9/f+SJw6lDkmbJjI7Akywd8/Y3gXsnaytJmh1THoEnuRp4L7A4ySjwKeC9SY4BCngI+N1ZrFGSNIEpA7yqJrql8bJZqEWSNA3eiSlJjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWrUlAGe5PIkjye5d8y8hUluTrK5+/Wg2S1TkjReL0fgVwInjJt3EXBrVb0RuLX7XpI0h6YM8Kq6A3hy3OyTgTXd6TXAhwdclyRpCjMdAz+4qrZ2p38MHDxZwySrkmxIsmHbtm0z3J0kaby+T2JWVQH1KstXV9VIVY0sWbKk391JkrpmGuCPJVkK0P36+OBKkiT1YqYBvg5Y2Z1eCawdTDmSpF71chnh1cC3gCOTjCY5G/gs8MEkm4Hjuu8lSXNo3lQNquq0SRZ9YMC1SJKmwTsxJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJatSUzwNv1V2PPPWy98ccOaRCJGmWeAQuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1Kj9tjrwF/VD9e/NH3kicOrQ5L60FeAJ3kIeAbYCeyoqpFBFCVJmtogjsDfV1VPDGA7kqRpcAxckhrVb4AX8LUkG5OsmqhBklVJNiTZsG3btj53J0napd8Af3dVHQucCJyf5D3jG1TV6qoaqaqRJUuW9Lk7SdIufQV4VW3pfn0c+Crw9kEUJUma2owDPMkvJdl/1zTwIeDeQRUmSXp1/VyFcjDw1SS7tvPlqrppIFVJkqY04wCvqgeAtwywFknSNHgZoSQ1ygCXpEYZ4JLUKANckhplgEtSo9p5nOzYR8AOY31J2s00E+B3PfLUwNY/5tAD+y1HkobOIRRJapQBLkmNMsAlqVEGuCQ1ygCXpEY1cxXKbmm6n27/apcy9rK+JI3hEbgkNcoAl6RGGeCS1CgDXJIatVeexHzZbfVHDrEQSeqDR+CS1Ki98gj8ZSa7FLCX+dPdviQN0F4f4A6nSGqVQyiS1CgDXJIaZYBLUqP6CvAkJyT5YZL7k1w0qKIkSVOb8UnMJPsAlwIfBEaBv0+yrqo2Daq4uXbLpsdenF786EsnN5/Y+dL84/YZzL7Gf0ScJ1AlTVc/V6G8Hbi/qh4ASHINcDLQbID3YrLP5hwb8mON/UUgSYPUT4AfAjwy5v0o8GvjGyVZBazqvn02yQ9nuL/FwBMzXLcB/3GyBXt4vye0N/YZ9s5+7419hun3+/UTzZz168CrajWwut/tJNlQVSMDKKkpe2O/98Y+w97Z772xzzC4fvdzEnMLcOiY98u68yRJc6CfAP974I1JDkuyL3AqsG4wZUmSpjLjIZSq2pHko8D/BfYBLq+qHwysslfqeximUXtjv/fGPsPe2e+9sc8woH6nqgaxHUnSHPNOTElqlAEuSY3a7QJ8qtvzk/xCkmu7y7+dZPncVzlYPfT595NsSnJ3kluTTHhNaGt6fRRDkt9KUkmav9yslz4n+Uj3+/2DJF+e6xpnQw8/47+c5LYk3+v+nJ80jDoHKcnlSR5Pcu8ky5PkL7v/JncnOXbaO6mq3eZF52Toj4DDgX2B7wNHj2vzX4AvdadPBa4ddt1z0Of3Aft1p89rvc+99rvbbn/gDuBOYGTYdc/B9/qNwPeAg7rvXzfsuueo36uB87rTRwMPDbvuAfT7PcCxwL2TLD8JWA8EeAfw7enuY3c7An/x9vyq+hmw6/b8sU4G1nSnrwc+kCRzWOOgTdnnqrqtqn7afXsnnWvuW9fL9xrgvwF/Bjw3l8XNkl76/J+BS6vqJwBV9fgc1zgbeul3Af+qO30A8Ogc1jcrquoO4MlXaXIycFV13AkcmGTpdPaxuwX4RLfnHzJZm6raATwNLJqT6mZHL30e62w6v7VbN2W/u39SHlpV/2cuC5tFvXyv3wS8Kck3k9yZ5IQ5q2729NLvTwO/k2QUuBH42NyUNlTT/b//Cnv9R6q1JMnvACPAbwy7ltmW5DXA54Ezh1zKXJtHZxjlvXT+0rojya9W1Z7+VLTTgCur6nNJ3gn8dZJfqaqfD7uw3dnudgTey+35L7ZJMo/On1vb56S62dHTIwmSHAdcDKyoqufnqLbZNFW/9wd+Bbg9yUN0xgjXNX4is5fv9SiwrqpeqKoHgX+gE+gt66XfZwPXAVTVt4AFdB74tCfr+3Eku1uA93J7/jpgZXf6FODr1T0j0Kgp+5zkrcD/pBPee8KYKEzR76p6uqoWV9XyqlpOZ+x/RVVtGE65A9HLz/f/pnP0TZLFdIZUHpjLImdBL/3+R+ADAEmOohPg2+a0yrm3DjijezXKO4Cnq2rrtLYw7DO1k5yZ/Qc6Z60v7s77Ezr/eaHzjf1b4H7gO8Dhw655Dvp8C/AYcFf3tW7YNc9Fv8e1vZ3Gr0Lp8XsdOkNHm4B7gFOHXfMc9fto4Jt0rlC5C/jQsGseQJ+vBrYCL9D5y+ps4Fzg3DHf60u7/yb3zOTn21vpJalRu9sQiiSpRwa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJatT/B2ei7KZgcRelAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAds0lEQVR4nO3deXxddZ3/8dc7W9MlbUqbltK9UKBlh8i+b0NBqRsKDiKIoCjqT5T5oaOo6DiIPxl1BkdQEBwXEETsSFlkd6HYsNOWQimFtkgbuu9pks/vj3OTJmna3LY59zY97+fjkUfO+Z7vPfdzkva+8z2rIgIzM8uukmIXYGZmxeUgMDPLOAeBmVnGOQjMzDLOQWBmlnFlxS5gWw0ePDjGjBlT7DLMzHqUp59++p2IqOlsWY8LgjFjxlBXV1fsMszMehRJb2xpmXcNmZllnIPAzCzjHARmZhnnIDAzyzgHgZlZxqUWBJJukbRY0ktbWC5JP5I0R9ILkg5NqxYzM9uyNEcEtwJnbGX5JGB87utS4L9TrMXMzLYgtSCIiCeApVvpMhn4RSSmAdWShqVVz/R5S7n+wdk0NDan9RZmZj1SMY8RDAfmt5lfkGvbjKRLJdVJqquvr9+uN3vmjWX86JE5NDY7CMzM2uoRB4sj4qaIqI2I2pqaTq+QNjOz7VTMIFgIjGwzPyLXZmZmBVTMIJgCXJA7e+hIYEVE/KOI9ZiZZVJqN52T9BvgRGCwpAXA14FygIj4CTAVOBOYA6wFLkqrFjMz27LUgiAizutieQCfSev9zcwsPz3iYLGZmaXHQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzy7hUg0DSGZJmS5oj6apOlo+S9KikZyW9IOnMNOsxM7PNpRYEkkqBG4BJwETgPEkTO3T7KvDbiDgEOBf4cVr1mJlZ59IcERwOzImIuRHRANwOTO7QJ4D+uekBwFsp1mNmZp1IMwiGA/PbzC/ItbX1DeB8SQuAqcBnO1uRpEsl1Umqq6+vT6NWM7PMKvbB4vOAWyNiBHAm8D+SNqspIm6KiNqIqK2pqSl4kWZmu7I0g2AhMLLN/IhcW1sXA78FiIgngUpgcIo1mZlZB2kGwXRgvKSxkipIDgZP6dDnTeAUAEkTSILA+37MzAootSCIiEbgcuABYBbJ2UEzJF0j6excty8Cl0h6HvgNcGFERFo1mZnZ5srSXHlETCU5CNy27eo20zOBY9KswczMtq7YB4vNzKzIHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxqQaBpDMkzZY0R9JVW+jzIUkzJc2Q9Os06zEzs82VpbViSaXADcBpwAJguqQpETGzTZ/xwJeBYyJimaQhadVjZmadS3NEcDgwJyLmRkQDcDswuUOfS4AbImIZQEQsTrEeMzPrRJpBMByY32Z+Qa6trb2BvSX9VdI0SWd0tiJJl0qqk1RXX1+fUrlmZtmUVxBIOkbSnyS9ImmupNclze2G9y8DxgMnAucBP5VU3bFTRNwUEbURUVtTU9MNb2tmZi3yPUZwM/AF4GmgKc/XLARGtpkfkWtrawHwVERsBF6X9ApJMEzP8z3MzGwH5btraEVE3BcRiyNiSctXF6+ZDoyXNFZSBXAuMKVDn3tIRgNIGkyyq6g7RhpmZpanfEcEj0r6HnA3sKGlMSKe2dILIqJR0uXAA0ApcEtEzJB0DVAXEVNyy06XNJNkpHFlHgFjZmbdKN8gOCL3vbZNWwAnb+1FETEVmNqh7eo20wFckfsyM7MiyCsIIuKktAsxM7PiyPesoQGSrm85hVPS9yUNSLs4MzNLX74Hi28BVgEfyn2tBH6eVlFmZlY4+R4j2DMiPtBm/puSnkujIDMzK6x8RwTrJB3bMiPpGGBdOiWZmVkh5TsiuAy4LXdcQMBS4MK0ijIzs8LJ96yh54CDJPXPza9MtSozMyuYrQaBpPMj4peSrujQDkBEXJ9ibWZmVgBdjQj65r5XpV2ImZkVx1aDICJuzH3/ZmHKMTOzQsv3grLrJPWXVC7pYUn1ks5PuzgzM0tfvqePnp47QPxuYB6wF3BlWkWZmVnh5BsELbuQzgLujIgVKdVjZmYFlu91BH+U9DLJRWSXSaoB1qdXlpmZFUpeI4KIuAo4GqjNPU1sDZs/iN7MzHqgrq4jODkiHpH0/jZtbbvcnVZhZmZWGF3tGjoBeAR4TyfLAgeBmVmP19V1BF/Pfb+oMOWYmVmh5XsdwXckVbeZHyjp2+mVZWZmhZLv6aOTImJ5y0xELAPOTKckMzMrpHyDoFRSr5YZSb2BXlvpb2ZmPUS+1xH8CnhYUsvjKS8CbkunJDMzK6R8n0fwXUnPA6fmmr4VEQ+kV5aZmRVKviMCgFlAY0Q8JKmPpKqIWJVWYWZmVhj5njV0CXAXcGOuaThwT1pFmZlZ4eR7sPgzwDHASoCIeBUYklZRZmZWOPkGwYaIaGiZkVRGcmWxmZn1cPkGweOSvgL0lnQacCfwv+mVZWZmhZJvEPxfoB54EfgkMBX4alpFmZlZ4XR51pCkUmBGROwL/DT9kszMrJC6HBFERBMwW9KoAtRjZmYFlu+uoYHAjNyD66e0fHX1IklnSJotaY6kq7bS7wOSQlJtvoWbmVn3yPeCsq9t64pzu5RuAE4DFgDTJU2JiJkd+lUBnwee2tb3MDOzHdfVE8oqgU8Be5EcKL45IhrzXPfhwJyImJtb1+0kj7ec2aHft4DvAlduQ91mZtZNuto1dBtQSxICk4Dvb8O6hwPz28wvyLW1knQoMDIi7t3aiiRdKqlOUl19ff02lGBmZl3patfQxIg4AEDSzcDfu+uNJZUA1wMXdtU3Im4CbgKora31hWxmZt2oqxHBxpaJbdgl1GIhMLLN/IhcW4sqYH/gMUnzgCOBKT5gbGZWWF2NCA6StDI3LZIri1fmpiMi+m/ltdOB8ZLGkgTAucBHWhZGxApgcMu8pMeAL0VE3TZvhZmZbbeuHl5fur0rjohGSZcDDwClwC0RMUPSNUBdRHR5+qmZmaVvW55HsM0iYirJ7Sjatl29hb4nplmLmZl1Lt8LyszMbBflIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4wrS3Plks4AfgiUAj+LiGs7LL8C+ATQCNQDH4+IN9KsqVjeXLKWHzz8Co1NAUCfilKumrQv1X0qilyZmWVdaiMCSaXADcAkYCJwnqSJHbo9C9RGxIHAXcB1adVTbI+9spi7n1nIs/OXMX3eUm6fPp9n5y8vdllmZqmOCA4H5kTEXABJtwOTgZktHSLi0Tb9pwHnp1hPt2luDh6cuYg1GxoBKCsVJ+87hKrK8nb9Fq1cz19efQeA3z2zEIB7Pn0Mby5dy/t+/LfCFm1mtgVpBsFwYH6b+QXAEVvpfzFwX2cLJF0KXAowatSo7qpvu724cAWf+uXT7dq+etYEPnHcuHZtP3z4VX791Jvt2laub0y9PjOzbZHqMYJ8STofqAVO6Gx5RNwE3ARQW1sbBSytUw1NzQBc/6GDOHDEAE69/onWtl9Oe4PHX6kH4E8zFwHwxJUn8fO/vc7P/zqP3ftXsnxtQ7v1rd/YxNqGptb5gX3KkVSITTEzSzUIFgIj28yPyLW1I+lU4F+BEyJiQ4r1bLN576zhoVnJh/m0uUt5aNYihlT1YvGqpMyGxmZGDOwDwHX3z+aNd9ZyR10yCJowrH/rekYN6sPX37MfX3/Pfpu9R3NzcML3HmXRyk2bfsFRo7lm8v6pbZeZWVtpBsF0YLyksSQBcC7wkbYdJB0C3AicERGLU6xlu/znI3P43TML2rUdOmogT85dwop1G5m3ZC0bGptbl7WMBADu+/xxjLnq3i7fI4BFKzdw0j41nLjPEP77sdd4a/n6btsGM7OupHbWUEQ0ApcDDwCzgN9GxAxJ10g6O9fte0A/4E5Jz0maklY926OpuZkRA3vzwjdOZ3C/5DTPfz1rAp88Ydxmfasqy5j2lVPatT34heP52QW1m/VdsyHZDXTRz6dz4c//DsAhowbysaPH8PbK9Tw0axHHX/cox1/3KP9y1/PdvVlmZu2keowgIqYCUzu0Xd1m+tQ03787lJaI/pXl9CorbW1778HDue7+2Zz7rpFbeSXsPbSKvYdWbdb+6uJVrdOr1jdy2OiBHLXnoHZ9Dhs9kGffXMZjs+s7vtzMrFvtFAeLe5o9qnsz79qzAFixbmO7ZX/87LH073Aa6ZZcePQYvnH25scNAP7jwwfz5btf4OFZO90eMzPbxTgIOpg2dwnP5S70uue5t7rs378y+RF+530HALD/8AFdvmbs4L4AjB/ab7NltaMHti43MysEB0EHV//hJV5ZtHqz9iv/aR/+zx3PsUd173btklpHB/k6cZ8h3Pu5Y9lvj81D467Ljt62gs3MdpCDAPj9swtYsHQdAK8sWs2eNX3542ePY8LV97f2ee8hw3nvIcO77T07C4GOHpyxiCVrGph8w18BGFrVixv++VDKS32vQDPrPpkPgrUNjXzhjvZn5rxWv4beFaX8/SunsLzDMYBCWrImufCsunc5by1fx4Pzl7N0TQND+1cWrSYz2/Vk/k/L5tx1yldN2pc5/zap3bIh/Ss7Peun0G77+OFcdMzYYpdhZruozI4I5r2zhtUbGltvHCegbCfb5fL7Tx/NjLdWFrsMM9vFZTIIXlq4gnf/51/atd359AI+ecKeRaqoc4eMGsghowYWuwwz28XtXH8CF8jK9cl+/y+dvjf//v7ktM+VRTwWYGZWTJkcEbSoHbNbp+fs7zHAB2PNLDsyHQSd2dZrAgql5bYUH77xSYb0r6R3eSnXfuAAhg3o3cUrzcy2LpO7hnqiunnLANjYFKxraOLxV+p5aWFyIDkiaGre9GVmti08IughTp0wlBcXruCPnz2WhcvXtTvYfer1j/Na/Zp2/ftXlrU+Da28NHnIzeB+vXjoihPo28u/djPbJPMjgkF9k9tLX3r85reW3pl8/tTxzLv2LAb2raB+dfIQm0t+UceEr92fXABXXsoVp+3d2v/9h45onb7kuHEcu9dg/rFiPcs6PB3NzCzzfxqWlZbstMcFtqQ+9zSz8lLx0aNGUyLxkcNHMWpQHzY2NVPdp4KLjx3LpP1356nXl/K5U8bznamzeHR2PZP/66+MGNibXuWl/L8PHsSoQX2KvDVmVmyZD4Ke6L2HDOfqKS/x5385mZqqXu2WffH0fVqnjxg3iCPGJc85mJm7MK05gvLSEv7++lJmvLXCQWBm3jXUE1WUlfDytyZtFgJbc90HD+SIsbvx7NWn8+33+XnIZraJgyAj9qjuzR2fPKrYZZjZTshBYGaWcZk5RvDIy8kjHyde/QAVZUn+lUjFLKnoZi9axbiafpQIxtX0o7Qk2z8Ps6zKTBDMWZw8dezDtSPp37uMvr3KOHBE1w+H2RX9Y/l6AH7w0Kv84KFXW9v3GVrF7EXJFczlpSIChlVX8utPHAlAVWUZ1X0qCl+wmaUqM0HQ4soz9mFwv/wPsu6KJgzrD8C57xrJ8XvX8OlfPQMkz1JuCYJLjhvHjx97jflL13HcdY9ucV17D+1HicSXTt+HUycOZf3GJuYvXdu6fEhVJQP6lKe4NWa2ozIXBAa7D6hsd+3EE1eeRE1VL3pXlLbrt2LdRn711Jt874MHctfTC3jq9aV8uHYkqzc0cu+L/wBgz5p+3PfS23ziF3VUlJXQ0Ni82ft9/5yDWLa2gTvrFnDUnoNY19DEHXXzqanqRanE2yuTEcoBwwfw4sIVABw8sprn5i8HktHJxqbk1hnHjR9M34oy7p/xNn0rStmjujeLV23gnMNGMOmAYUQE1X0qqM6FT79eZVSWl25Wk5ltooiedW+a2traqKur2+bXHfatP7FkTQN1Xz018yOC7jbmqnsB+NQJe/KTx1+jX68yrv3AAVz+62c77d+nopSGxmb226M/++7enzvq5gNw8r5DWo/lHL93DU+8Ug/Ap0/ck1v/No+1DU1AcquMd3JXV595wO5MffHtrdZ340cPA2BDYzMHj6hGgvLSEnb3XWYtQyQ9HRG1nS5zEFhanp+/nNunv8llJ+xFcwSvLFrF6fvtvt3rW7V+I1WVm+9m+vLdL9KrrIST9h3Ctfe9zNz61Xz1rAl87Q8ztrq+EsHpE3dHgguOGsNRew7a7trMdnZbCwLvGrLUHDSymoNGVrfOj+nk2Q/borMQAFofLgRwwt41rdPHja/h1r/N45zaESxd08BHb/47177/AFZvaOTb986iOeD1d9Ywe9Eq7nvpbSrLS1i/Mdm1Na6mL3NzN/LbrW8FS9dsukdTiZJnXU8+eA+O2Wswc+vXMGxAJUP7JyOMCcOqGD1ox7bVrJAcBLbLGjO4L984e7/W+bbHRT5x3KabDB797w/zzpoGPnbUGG58Yi4AE4f1580la2lsDibtvzuLVm7goVmLOP/IUTQ2BbdPn88fnnuLPzz3Vqfvfd7ho3h7xTqmzV3KxceOBWD5ugbOfdcoyktL2NDYxOB+vWg5g3loVSUlPn3XisRBYJn3ty+f0jr95TMn5PWa0yYOZfSgvlSWl/Dka0voU1HG2MF9OfNHfwbgoVmLqF+VHMf4r0fntL7ul9Pe3OI63zVmICIXBu2/sXRNA68uXs3w6t5U9yknAoLkWRQtkragpak5grdXrOfY8YMpKylhY1Mzlx4/jgG9yykpEeMG90UZv5bGEg4Cs+1wyoShrdPn1G66cV/bUUfLA4PKSktobGrm3JumceExYyiR+Ld7Z/Hug4YxdlBf/vDcW0i0foAHkXt9yzxsbEp2WVVVljGs9SC3kJKwUGtw5NoEry1ew279KpizeDXzlqylqTl4cOaidtsxfkhy+q+UXGBZUkJuXpS0tAlWrW/k5bdXUVPVi8H9etHcHDRH8tXyLIx9d6+iOZIgao7IBVUyvXR1A6s2JM/HqO5TTnPzpuWrc+3jh/Sjqc3rG5uCRSvXc9K+Q+hdXtq6rSUSdz+7kNGD+rDP0CqaI/lZJ/XQrobWtuZg9YZGxtX0ZeRufVp/TiVq8zMDpJbpZLuTn+WmtpZ+AQyv7s0e1ZWb/a5ap3MTLVHdEtRtG1uWVfcpZ7e+Fe0CftN6NgX8bv0q6L+FXaQ7wkFglhJJlOUeClRWWsJdlx3duuzMA4a1Tp97+KjUa4kIHn+lvvVD9/fPLKSyojT5AG2m9QNzSx+oLSOH6t7ljBjYmxJBaUkSGC1BMHpQ8gFbUtI2kJIP1I1NzUx98W1OmziUPQZUtn64lkjc/JfXARg/tB+SKM29Zt6StTRH8Gru2paWD9mWD9M3lqyld3npVgOsZX59UzMvv72KOYtXU1qidh+ubUNrZ/ft9+7P+UeO7vb1OgjMMkASJ+4zpHX+3Qfu0W3rvuEjO/b6r717YvcU0g1a/hJv+au8uU1gtA2PN5asbX1AVMvOtY6jsrbL2MqyBcvWsb6xKQmy3HpEMvRoHaXk2g9uc/JFd0o1CCSdAfwQKAV+FhHXdljeC/gFcBiwBPhwRMxLsyYzsy1pGank5rbYb+Ie/bvtPY/otjVtv9TuPiqpFLgBmARMBM6T1DH6LwaWRcRewH8A302rHjMz61yat6E+HJgTEXMjogG4HZjcoc9k4Lbc9F3AKUr5NAafI2Fm1l6aQTAcmN9mfkGurdM+EdEIrAA2u7xT0qWS6iTV1dfXb1cx3//QQVT1KmOQryo2M2unRzyYJiJuiojaiKitqanp+gWdOHGfIbz4zX/q5srMzHq+NINgITCyzfyIXFunfSSVAQNIDhqbmVmBpBkE04HxksZKqgDOBaZ06DMF+Fhu+oPAI9HT7oJnZtbDpXb6aEQ0SroceIDk9NFbImKGpGuAuoiYAtwM/I+kOcBSkrAwM7MCSvU6goiYCkzt0HZ1m+n1wDlp1mBmZlvXIw4Wm5lZehwEZmYZ5yAwM8s4B4GZWcb1uGcWS6oH3tjOlw8G3unGcnoCb3M2eJuzYUe2eXREdHpFbo8Lgh0hqW5LD2/eVXmbs8HbnA1pbbN3DZmZZZyDwMws47IWBDcVu4Ai8DZng7c5G1LZ5kwdIzAzs81lbURgZmYdOAjMzDJulwwCSWdImi1pjqSrOlneS9IdueVPSRpT+Cq7Vx7bfIWkmZJekPSwpNHFqLM7dbXNbfp9QFJI6vGnGuazzZI+lPtdz5D060LX2N3y+Lc9StKjkp7N/fs+sxh1dhdJt0haLOmlLSyXpB/lfh4vSDp0h980InapL5JbXr8GjAMqgOeBiR36fBr4SW76XOCOYtddgG0+CeiTm74sC9uc61cFPAFMA2qLXXcBfs/jgWeBgbn5IcWuuwDbfBNwWW56IjCv2HXv4DYfDxwKvLSF5WcC95E8gv1I4Kkdfc9dcURwODAnIuZGRANwOzC5Q5/JwG256buAUyT15Ofad7nNEfFoRKzNzU4jeWJcT5bP7xngW8B3gfWFLC4l+WzzJcANEbEMICIWF7jG7pbPNgfQPzc9AHirgPV1u4h4guT5LFsyGfhFJKYB1ZKG7ch77opBMByY32Z+Qa6t0z4R0QisAAYVpLp05LPNbV1M8hdFT9blNueGzCMj4t5CFpaifH7PewN7S/qrpGmSzihYdenIZ5u/AZwvaQHJ808+W5jSimZb/793KdUH09jOR9L5QC1wQrFrSZOkEuB64MIil1JoZSS7h04kGfU9IemAiFhe1KrSdR5wa0R8X9JRJE893D8imotdWE+xK44IFgIj28yPyLV12kdSGclwcklBqktHPtuMpFOBfwXOjogNBaotLV1tcxWwP/CYpHkk+1Kn9PADxvn8nhcAUyJiY0S8DrxCEgw9VT7bfDHwW4CIeBKoJLk5264qr//v22JXDILpwHhJYyVVkBwMntKhzxTgY7npDwKPRO4oTA/V5TZLOgS4kSQEevp+Y+himyNiRUQMjogxETGG5LjI2RFRV5xyu0U+/7bvIRkNIGkwya6iuYUsspvls81vAqcASJpAEgT1Ba2ysKYAF+TOHjoSWBER/9iRFe5yu4YiolHS5cADJGcc3BIRMyRdA9RFxBTgZpLh4xySgzLnFq/iHZfnNn8P6AfcmTsu/mZEnF20ondQntu8S8lzmx8ATpc0E2gCroyIHjvazXObvwj8VNIXSA4cX9iT/7CT9BuSMB+cO+7xdaAcICJ+QnIc5ExgDrAWuGiH37MH/7zMzKwb7Iq7hszMbBs4CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8CsE5KaJD0n6SVJ/yupupvXPy93nj+SVnfnus22lYPArHPrIuLgiNif5FqTzxS7ILO0OAjMuvYkuZt6SdpT0v2Snpb0Z0n75tqHSvq9pOdzX0fn2u/J9Z0h6dIiboPZFu1yVxabdSdJpSS3L7g513QT8KmIeFXSEcCPgZOBHwGPR8T7cq/pl+v/8YhYKqk3MF3S73rylb62a3IQmHWut6TnSEYCs4A/SeoHHM2m23QA9Mp9Pxm4ACAimkhubQ7wOUnvy02PJLkBnIPAdioOArPOrYuIgyX1IbnPzWeAW4HlEXFwPiuQdCJwKnBURKyV9BjJDdHMdio+RmC2Fbmnun2O5MZma4HXJZ0Drc+OPSjX9WGSR4AiqVTSAJLbmy/LhcC+JLfCNtvpOAjMuhARzwIvkDwA5Z+BiyU9D8xg02MTPw+cJOlF4GmSZ+feD5RJmgVcS3IrbLOdju8+amaWcR4RmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZx/x+AvhTI/UJgtwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_col = \"pred_Option\"\n",
    "col = \"Option\"\n",
    "hist_multi_plot(df, pred_col, col, bins=np.arange(0,1, 0.01))\n",
    "precision, recall, _ = precision_recall_curve(df[col], df[pred_col])\n",
    "pr_display = PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "average_precision = average_precision_score(df[col], df[pred_col])\n",
    "print(\"Average Precision: \", average_precision)\n",
    "strawman_ce = log_loss(df[col], np.ones(df[col].shape)*means[col])\n",
    "print(\"Strawman Crossentropy\", strawman_ce)\n",
    "bce = log_loss(df[col], df[pred_col])\n",
    "print(\"Binary Crossentropy\", bce)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Precision:  0.1704913368401367\n",
      "Strawman Crossentropy 0.2087954929155983\n",
      "Binary Crossentropy 0.19396977538777002\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXW0lEQVR4nO3df5RU5X3H8fcnAt2jpSiwGmVRSDGcaFqtXY05tdYkxh/bdElbm0Ibf0Qs1Zocm/6Kja0a0/a0pyftaYsnlFb8kUQ0mho4CaBokkPThuiSosGNBmJId1cCyyoYm6BCvv1jLmRYZ9i7c2dnd575vM6Zw517n7nP87Dw2Weee+cZRQRmZpauN4x3A8zMbGw56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegN6tC0nZJF47h+W+V9OmxOr/ZQQ56swIkHStphaTvS/qBpG9LunG822VWbtJ4N8CsESRNioj9Y3DqfwSOAd4C7AXeDLx1DOoxq5lH9NbUsumVP5fUK+lFSXdKapN0gaR+SR+R9H3gTklvkHSjpO9IGpL0WUnTy851uaTvZcduytmEs4F7I+LFiPhxRDwTEQ+WnfOfJPVJeknSJkm/fIS+nCvpvyXtkfSkpAtq/XsxK+egtxT8LnAx8LOURtR/ke1/IzAdOAVYAnwIeC/wK8BJwIvA7QCSTgM+CVyeHZsBdOSoeyPw15I+IOnUCsefAM7M2nEv8ICktuGFJM0Cvgj8VVb2T4DPSWrP0QazI3LQWwqWRkRfRLwA/DWwKNv/Y+CWiHglIn4EXAvcFBH9EfEKcCtwmaRJwGXAFyJiQ3bsL7PXj+RDwGeADwK9krZJuvTgwYj4dEQMRcT+iPgE8FPA/ArneT+wJiLWZO8M1gM9QNeo/zbMhnHQWwr6yra/R2lEDjAYEfvKjp0CPJRNjewBvgUcAE7IXnPoPBHxf8DQSBVHxI8i4m8i4hcpvQv4LKVR+3QASX8i6VuS9mZ1TgNmVjjVKcBvHWxbVvY84MQ8fwFmR+KgtxTMLts+GXg+2x6+NGsfcGlEHFv2aIuIAWBH+XkkHU0puHOLiJeAv6F0cXZuNh//Z8D7gOMi4lhKF2xV4eV9wKeGte2YiPjb0bTBrBIHvaXgekkd2Sj6JuD+KuWWUZpPPwVAUrukBdmxB4H3SDpP0hTgNnL8/5D0l5LOljQlm3u/AdgDPAtMBfYDg8AkSTcDP1PlVJ8Gfk3SxZKOKrugnOc6gdkROegtBfcCjwDPAd+hdEGzkn8CVgOPSPoBpQupbwOIiKeB67Nz7aB0obY/R90B3AnspvRO4t3Ar0bEy8DDwDrg25SmlPZx+DTTT04S0QcsAD5K6RdDH/Cn+P+o1YH8xSPWzCRtB66JiEfHuy1mE5VHC2ZmiXPQm41A0lpJL1d4fHS822aWh6duzMwS5xG9mVniJuSiZjNnzow5c+aMdzPMzJrGpk2bdkdExSUzJmTQz5kzh56envFuhplZ05D0vWrHPHVjZpY4B72ZWeIc9GZmiZuQc/RmZvX22muv0d/fz759+0YuPIG1tbXR0dHB5MmTc7/GQW9mLaG/v5+pU6cyZ84cpEoLiE58EcHQ0BD9/f3MnTs39+s8dWNmLWHfvn3MmDGjaUMeQBIzZswY9bsSB72ZtYxmDvmDaumDg97MLHGeozezlvRo7866nu/C004Yscy6deu44YYbOHDgANdccw033njjYcdfeeUVrrjiCjZt2sSMGTO4//77qccqAS05on+0d+ehh5lZIxw4cIDrr7+etWvX0tvby8qVK+nt7T2szB133MFxxx3Htm3b+PCHP8xHPvKRutTdkkFvZtZojz/+OPPmzeNNb3oTU6ZMYeHChaxateqwMqtWreLKK68E4LLLLuOxxx6jHisM5/lOzBWSdknaUrbvfkmbs8d2SZurvHa7pG9m5bx4jZm1rIGBAWbP/sn32Hd0dDAwMFC1zKRJk5g2bRpDQ0OF684zR38XsBS45+COiPjtg9uSPkHpm+2reUdE7K61gWZmVsyII/qI2AC8UOmYSvf5vA9YWed2mZklZdasWfT1/eS74fv7+5k1a1bVMvv372fv3r3MmDGjcN1F5+h/GdgZEVurHA/gEUmbJC050okkLZHUI6lncHCwYLPMzCaWs88+m61bt/Ld736XV199lfvuu4/u7u7DynR3d3P33XcD8OCDD/LOd76zLvf+F729chFHHs2fFxEDko4H1kt6JnuH8DoRsRxYDtDZ2envNzSzMZXndsh6mjRpEkuXLuXiiy/mwIEDXH311Zx++uncfPPNdHZ20t3dzeLFi7n88suZN28e06dP57777qtP3bW+UNIk4DeAX6xWJiIGsj93SXoIOAeoGPRmZqnr6uqiq6vrsH233Xbboe22tjYeeOCButdbZOrmQuCZiOivdFDSMZKmHtwGLgK2VCprZmZjJ8/tlSuBrwHzJfVLWpwdWsiwaRtJJ0lakz09AfiqpCeBx4EvRsS6+jXdzMzyGHHqJiIWVdl/VYV9zwNd2fZzwBkF22dmZgX5k7FmZolz0JuZJc5Bb2aWOC9TbGat6dm19T3f/EtHLHL11VfzhS98geOPP54tW15/E2JEcMMNN7BmzRqOPvpo7rrrLs4666zCTfOI3sysQa666irWrat+8+HatWvZunUrW7duZfny5Vx33XV1qddBb2bWIOeffz7Tp0+venzVqlVcccUVSOLcc89lz5497Nixo3C9Dnozswkiz1LGtXDQm5klruUvxpZ/nWCjFzkyMyuXZynjWnhEb2Y2QXR3d3PPPfcQEWzcuJFp06Zx4oknFj5vy4/ozaxF5bgdst4WLVrEV77yFXbv3k1HRwcf+9jHeO211wC49tpr6erqYs2aNcybN4+jjz6aO++8sy71OujNzBpk5cojfxmfJG6//fa61+upGzOzxDnozcwS56A3s5YR0fzfUlpLHxz0ZtYS2traGBoaauqwjwiGhoZoa2sb1et8MdbMWkJHRwf9/f0MDg6Od1MKaWtro6OjY1SvcdCbWUuYPHkyc+fOHe9mjAtP3ZiZJS7Pl4OvkLRL0payfbdKGpC0OXt0VXntJZKelbRN0o31bLiZmeWTZ0R/F3BJhf3/GBFnZo81ww9KOgq4HbgUOA1YJOm0Io01M7PRGzHoI2ID8EIN5z4H2BYRz0XEq8B9wIIazmNmZgUUmaP/oKSnsqmd4yocnwX0lT3vz/aZmVkD1Rr0nwR+FjgT2AF8omhDJC2R1COpp9lvfzIzm0hqCvqI2BkRByLix8C/UZqmGW4AmF32vCPbV+2cyyOiMyI629vba2mWmZlVUFPQSypfIPnXgdd/nTk8AZwqaa6kKcBCYHUt9ZmZWe1G/MCUpJXABcBMSf3ALcAFks4EAtgO/H5W9iTg3yOiKyL2S/og8DBwFLAiIp4ek16YmVlVIwZ9RCyqsPuOKmWfB7rKnq8BXnfrpZmZNY4/GWtmljivdVPGXxRuZinyiN7MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxLXMMsXlSxCbmbUSj+jNzBLnoDczS5yD3swscSMGvaQVknZJ2lK27+8lPSPpKUkPSTq2ymu3S/qmpM2SeurZcDMzyyfPiP4u4JJh+9YDb42Inwe+Dfz5EV7/jog4MyI6a2uimZkVMWLQR8QG4IVh+x6JiP3Z041Axxi0zczM6qAec/RXA2urHAvgEUmbJC050kkkLZHUI6lncHCwDs0yMzMoGPSSbgL2A5+pUuS8iDgLuBS4XtL51c4VEcsjojMiOtvb24s0y8zMytQc9JKuAt4D/G5ERKUyETGQ/bkLeAg4p9b6zMysNjUFvaRLgD8DuiPih1XKHCNp6sFt4CJgS6WyZmY2dvLcXrkS+BowX1K/pMXAUmAqsD67dXJZVvYkSWuyl54AfFXSk8DjwBcjYt2Y9MLMzKoaca2biFhUYfcdVco+D3Rl288BZxRqnZmZFeZPxpqZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZonLFfSSVkjaJWlL2b7pktZL2pr9eVyV116Zldkq6cp6NdzMzPLJO6K/C7hk2L4bgcci4lTgsez5YSRNB24B3gacA9xS7ReCmZmNjVxBHxEbgBeG7V4A3J1t3w28t8JLLwbWR8QLEfEisJ7X/8IwM7MxVGSO/oSI2JFtfx84oUKZWUBf2fP+bN/rSFoiqUdSz+DgYIFmmZlZubpcjI2IAKLgOZZHRGdEdLa3t9ejWWZmRrGg3ynpRIDsz10VygwAs8ued2T7zMysQYoE/Wrg4F00VwKrKpR5GLhI0nHZRdiLsn1mZtYgeW+vXAl8DZgvqV/SYuBvgXdL2gpcmD1HUqekfweIiBeAjwNPZI/bsn1mZtYgk/IUiohFVQ69q0LZHuCasucrgBU1tc7MzArzJ2PNzBLnoDczS5yD3swscQ56M7PEOejNzBLnoDczS5yD3swscQ56M7PEOejNzBLnoDczS5yD3swscbnWuknNzOe/dGh790nvrLif06ot72Nm1lw8ojczS5yD3swscQ56M7PEOejNzBLnoDczS1xL3nVT7rA7bczMEuQRvZlZ4hz0ZmaJqznoJc2XtLns8ZKkPxxW5gJJe8vK3Fy8yWZmNho1z9FHxLPAmQCSjgIGgIcqFP3PiHhPrfWYmVkx9Zq6eRfwnYj4Xp3OZ2ZmdVKvoF8IrKxy7O2SnpS0VtLp1U4gaYmkHkk9g4ODdWqWmZkVDnpJU4Bu4IEKh78BnBIRZwD/Any+2nkiYnlEdEZEZ3t7e9FmmZlZph4j+kuBb0TEzuEHIuKliHg5214DTJY0sw51mplZTvUI+kVUmbaR9EZJyrbPyeobqkOdZmaWU6FPxko6Bng38Ptl+64FiIhlwGXAdZL2Az8CFkZEFKmzVv4ErJm1qkJBHxH/B8wYtm9Z2fZSYGmROszMrBh/MtbMLHEOejOzxDnozcwS1/LLFFf17NqfbM+/dPzaYWZWkEf0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJ8ydjq9jct+fQ9pnzx7EhZmYFeURvZpY4B72ZWeIc9GZmiXPQm5klzkFvZpY433WTh9emN7MmVnhEL2m7pG9K2iypp8JxSfpnSdskPSXprKJ1mplZfvUa0b8jInZXOXYpcGr2eBvwyexPMzNrgEbM0S8A7omSjcCxkk5sQL1mZkZ9RvQBPCIpgH+NiOXDjs8C+sqe92f7dpQXkrQEWAJw8skn16FZ8GjvzkPbM+tyRjOz5lOPoD8vIgYkHQ+sl/RMRGwY7UmyXxDLATo7O6MO7aobL4dgZs2s8NRNRAxkf+4CHgLOGVZkAJhd9rwj22dmZg1QKOglHSNp6sFt4CJgy7Biq4ErsrtvzgX2RsQOzMysIYpO3ZwAPCTp4LnujYh1kq4FiIhlwBqgC9gG/BD4QME6zcxsFAoFfUQ8B5xRYf+ysu0Ari9Sj5mZ1c6fjC2i/BOzw/kTtGY2QXitGzOzxDnozcwS56A3M0ucg97MLHG+GDtKmx9deWj7zNnH5ntRtWWOvfyxmTWAR/RmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJa7moJc0W9KXJfVKelrSDRXKXCBpr6TN2ePmYs01M7PRKrIe/X7gjyPiG5KmApskrY+I3mHl/jMi3lOgHjMzK6DmoI+IHcCObPsHkr4FzAKGB32yNvftOex57i8iMTNroLrM0UuaA/wC8PUKh98u6UlJayWdfoRzLJHUI6lncHCwHs0yMzPqEPSSfhr4HPCHEfHSsMPfAE6JiDOAfwE+X+08EbE8IjojorO9vb1os8zMLFMo6CVNphTyn4mI/xh+PCJeioiXs+01wGRJM4vUOZFt7ttz6GFmNlHUPEcvScAdwLci4h+qlHkjsDMiQtI5lH6xDNVa52jNfP5Ljaoqv/IvBK+2f7RfFJ7ny8drOa+ZJaHIXTe/BFwOfFPS5mzfR4GTASJiGXAZcJ2k/cCPgIUREQXqNDOzUSpy181XAY1QZimwtNY6zMysOH8y1swscQ56M7PEFZmjtxSN9qJwkYvIZtYQHtGbmSXOI/ox8mjvzkPbFx41jg0xs5bnEb2ZWeI8om+A8k/KeuEzM2s0B/0Yqfap3LqFfrVP2I4nX8g1m5Ac9OPII30zawTP0ZuZJc5Bb2aWOE/dTBCHTePMH8eGmFlyHPQT0OZHVx7aHvXc/ZEu0uZdzng0ry3apjxlqtXni7lmuXjqxswscQ56M7PEeeomAb5N08yOxEE/wTnEzawoT92YmSXOI/omVT7Sz7Mfhr0jKHonTCPVq62+e8daVHJBX7488MxxbMdYOFKIj3V9njYya16Fpm4kXSLpWUnbJN1Y4fhPSbo/O/51SXOK1GfjZ3PfnkMPM2suNY/oJR0F3A68G+gHnpC0OiJ6y4otBl6MiHmSFgJ/B/x2kQZb7eoV0oe9a3p+dKP+vO8S/G7CrH6KTN2cA2yLiOcAJN0HLADKg34BcGu2/SCwVJIiIgrUaxNUeTjvPlD5l0G18rXUMVrlS0tU/WXl5ScsQUWCfhbQV/a8H3hbtTIRsV/SXmAGsHv4ySQtAZZkT1+W9GyN7ZpZ6fyJa8U+w6j7/Tt1KjOu/LNuHaPt8ynVDkyYi7ERsRxYXvQ8knoiorMOTWoardhnaM1+t2KfoTX7Xc8+F7kYOwDMLnveke2rWEbSJGAaMFSgTjMzG6UiQf8EcKqkuZKmAAuB1cPKrAauzLYvA77k+Xkzs8aqeeomm3P/IPAwcBSwIiKelnQb0BMRq4E7gE9J2ga8QOmXwVgrPP3ThFqxz9Ca/W7FPkNr9rtufZYH2GZmafNaN2ZmiXPQm5klrmmDvhWXX8jR5z+S1CvpKUmPSap6X20zGanfZeV+U1JIavrb8PL0WdL7sp/305LubXQb6y3Hv++TJX1Z0v9k/8a7xqOd9SRphaRdkrZUOS5J/5z9nTwl6ayaKoqIpntQuvj7HeBNwBTgSeC0YWX+AFiWbS8E7h/vdjegz+8Ajs62r2v2Puftd1ZuKrAB2Ah0jne7G/CzPhX4H+C47Pnx493uBvR5OXBdtn0asH28212Hfp8PnAVsqXK8C1gLCDgX+Hot9TTriP7Q8gsR8SpwcPmFcguAu7PtB4F3SVID21hvI/Y5Ir4cET/Mnm6k9NmGZpfnZw3wcUprKe1rZOPGSJ4+/x5we0S8CBARuxrcxnrL0+cAfibbngY838D2jYmI2EDpjsRqFgD3RMlG4FhJJ462nmYN+krLL8yqViYi9gMHl19oVnn6XG4xpZFAsxux39nb2dkR8cVGNmwM5flZvxl4s6T/krRR0iUNa93YyNPnW4H3S+oH1gAfakzTxtVo/99XNGGWQLD6kfR+oBP4lfFuy1iT9AbgH4CrxrkpjTaJ0vTNBZTeuW2Q9HMRkfI60ouAuyLiE5LeTukzOm+NiB+Pd8MmumYd0bfi8gt5+oykC4GbgO6IeKVBbRtLI/V7KvBW4CuStlOax1zd5Bdk8/ys+4HVEfFaRHwX+Dal4G9Wefq8GPgsQER8DWgjve8XGi7X//uRNGvQt+LyCyP2WdIvAP9KKeSbfc72oCP2OyL2RsTMiJgTEXMoXZvojoie8WluXeT59/15SqN5JM2kNJXzXCMbWWd5+vy/wLsAJL2FUtAPNrSVjbcauCK7++ZcYG9E7BjtSZpy6iYm7vILYyZnn/8e+Gnggey68/9GRPe4NboOcvY7KTn7/DBwkaRe4ADwpxHRtO9Yc/b5j4F/k/RhShdmr2rywRuSVlL6hT0zu/ZwCzAZICKWUboW0QVsA34IfKCmepr878nMzEbQrFM3ZmaWk4PezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8T9P8FhOH4bnY8yAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3hV5Zn38e+dI4GEBEgAScCgBBBRASMqVAXUDmpHOlYddZxWp6NtPbS1ffu+dtpqa+d6a3XqdDrjjNpRW2dabW1tZeqB1hP1hBJEjgrEAJJwSiAkgRByuuePvRNCSMgOZO1Nsn6f68qVvdZ69tr3gsAvaz1rPY+5OyIiEl5JiS5AREQSS0EgIhJyCgIRkZBTEIiIhJyCQEQk5FISXUBv5ebmemFhYaLLEBHpV5YtW1bl7nldbet3QVBYWEhJSUmiyxAR6VfMbHN323RpSEQk5BQEIiIhpyAQEQk5BYGISMgpCEREQi6wIDCzx8xsp5mt7ma7mdlPzKzUzFaa2YygahERke4FeUbwM2D+EbZfAhRFv24G/iPAWkREpBuBBYG7/xnYfYQmC4AnPGIJkGNmJwRVz9JNu3ngj+tobG4N6iNERPqlRPYR5ANbOiyXR9cdxsxuNrMSMyuprKw8qg97b3M1P3mllOZWBYGISEf9orPY3R9x92J3L87L6/IJaREROUqJDIIKYGyH5YLoOhERiaNEBsFC4LPRu4fOAWrcfVsC6xERCaXABp0zsyeBOUCumZUDdwOpAO7+EPA8cClQCtQDNwZVi4iIdC+wIHD3a3vY7sCtQX2+iIjEpl90FouISHAUBCIiIacgEBEJOQWBiEjIKQhEREJOQSAiEnIKAhGRkFMQiIiEnIJARCTkFAQiIiGnIBARCTkFgYhIyCkIRERCTkEgIhJyCgIRkZBTEIiIhJyCQEQk5BQEIiIhpyAQEQk5BYGISMgpCEREQk5BICIScgoCEZGQUxCIiIScgkBEJOQUBCIiIacgEBEJOQWBiEjIKQhEREJOQSAiEnIKAhGRkAs0CMxsvpmtM7NSM7uzi+3jzOxVM1tuZivN7NIg6xERkcMFFgRmlgw8CFwCTAGuNbMpnZp9G/i1u08HrgH+Pah6RESka0GeEcwESt29zN0bgaeABZ3aODA0+job2BpgPSIi0oUggyAf2NJhuTy6rqPvAtebWTnwPHB7Vzsys5vNrMTMSiorK4OoVUQktBLdWXwt8DN3LwAuBf7LzA6ryd0fcfdidy/Oy8uLe5EiIgNZkEFQAYztsFwQXdfR54FfA7j728AgIDfAmkREpJMgg2ApUGRm480sjUhn8MJObT4GLgQws1OIBIGu/YiIxFFgQeDuzcBtwCLgAyJ3B60xs3vM7PJos68DN5nZCuBJ4AZ396BqEhGRw6UEuXN3f55IJ3DHdXd1eL0WmB1kDSIicmSJ7iwWEZEEUxCIiIScgkBEJOQUBCIiIacgEBEJOQWBiEjIKQhEREJOQSAiEnIKAhGRkFMQiIiEnIJARCTkFAQiIiGnIBARCTkFgYhIyCkIRERCTkEgIhJyCgIRkZBTEIiIhJyCQEQk5BQEIiIhpyAQEQk5BYGISMgpCEREQk5BICIScgoCEZGQUxCIiIScgkBEJOQUBCIiIacgEBEJOQWBiEjIBRoEZjbfzNaZWamZ3dlNm6vNbK2ZrTGzXwZZj4iIHC4lqB2bWTLwIHAxUA4sNbOF7r62Q5si4JvAbHevNrORQdUjIiJdC/KMYCZQ6u5l7t4IPAUs6NTmJuBBd68GcPedAdYjIiJdCDII8oEtHZbLo+s6mghMNLM3zWyJmc3vakdmdrOZlZhZSWVlZUDlioiEU0xBYGazzexPZrbezMrMbKOZlfXB56cARcAc4Frgp2aW07mRuz/i7sXuXpyXl9cHHysiIm1i7SN4FLgDWAa0xPieCmBsh+WC6LqOyoF33L0J2Ghm64kEw9IYP0NERI5RrJeGatz9BXff6e672r56eM9SoMjMxptZGnANsLBTm98TORvAzHKJXCrqizMNERGJUaxnBK+a2f3AM8CBtpXu/l53b3D3ZjO7DVgEJAOPufsaM7sHKHH3hdFtnzSztUTONL4RQ8CIiEgfijUIzo5+L+6wzoF5R3qTuz8PPN9p3V0dXjvwteiXiIgkQExB4O5zgy5EREQSI9a7hrLN7IG2WzjN7Edmlh10cSIiErxYO4sfA+qAq6NftcDjQRUlIiLxE2sfwcnu/pkOy98zs/eDKEhEROIr1jOC/Wb2ibYFM5sN7A+mJBERiadYzwi+BPw82i9gwG7ghqCKEhGR+In1rqH3gTPMbGh0uTbQqkREJG6OGARmdr27/7eZfa3TegDc/YEAaxMRkTjo6YxgSPR7VtCFiIhIYhwxCNz94ej378WnHBERibdYHyi7z8yGmlmqmb1sZpVmdn3QxYmISPBivX30k9EO4k8Bm4AJwDeCKkpEROIn1iBou4R0GfC0u9cEVI+IiMRZrM8R/MHMPiTyENmXzCwPaAiuLBERiZeYzgjc/U5gFlAcnU1sH4dPRC8iIv1QT88RzHP3V8zsig7rOjZ5JqjCREQkPnq6NHQB8Arwl11scxQEIiL9Xk/PEdwd/X5jfMoREZF4i/U5gv9vZjkdloeZ2T8GV5aIiMRLrLePXuLue9oW3L0auDSYkkREJJ5iDYJkM0tvWzCzDCD9CO1FRKSfiPU5gl8AL5tZ2/SUNwI/D6YkERGJp1jnI/ihma0ALoqu+r67LwquLBERiZdYzwgAPgCa3f0lMxtsZlnuXhdUYSIiEh+x3jV0E/Ab4OHoqnzg90EVJSIi8RNrZ/GtwGygFsDdNwAjgyoqEbbu2c+Lq7ezsnxPz41FRAaQWC8NHXD3xrbhJcwshciTxQPGN59ZxeL1laQmG2vvmU9qcqwZKSLSv8X6v91iM/sHIMPMLgaeBv4nuLLir6GpBYCmFqeldUBlnIjIEcUaBP8PqARWAV8Ange+HVRRIiISPz1eGjKzZGCNu08Gfhp8SSIiEk89nhG4ewuwzszGxaEeERGJs1gvDQ0D1kQnrl/Y9tXTm8xsvpmtM7NSM7vzCO0+Y2ZuZsWxFi4iIn0j1ruGvtPbHUcvKT0IXAyUA0vNbKG7r+3ULgv4CvBObz9DRESOXU8zlA0CvghMINJR/Ki7N8e475lAqbuXRff1FJHpLdd2avd94IfAN3pRd59pbXWufvhtVnR6fuDLTy7n7bJdfGJCLv/819MSUZqISFz0dGno50AxkRC4BPhRL/adD2zpsFweXdfOzGYAY939uSPtyMxuNrMSMyuprKzsRQk9a2xppWRzNVPGZDOzcHj7+iVlu6isO8CSsl19+nkiIsebni4NTXH30wDM7FHg3b76YDNLAh4Abuiprbs/AjwCUFxcHMhN/vNPHY3jvLtpdxC7FxE5bvV0RtDU9qIXl4TaVABjOywXRNe1yQKmAq+Z2SbgHGBhIjuMPRoxO2sPJKoEEZG46+mM4Awzq42+NiJPFtdGX7u7Dz3Ce5cCRWY2nkgAXANc17bR3WuA3LZlM3sN+D/uXtLro+gjT5dErmSdf/+riSpBRCTuepq8Pvlod+zuzWZ2G7AISAYec/c1ZnYPUOLuPd5+Gm+79jUetm5bTQPT7vkjzS3OT66dxrzJoxJQmYhIcHozH0GvufvzRIaj6Ljurm7azgmylqNx4ojBbN5Vz576yBWy0p17FQQiMuBoiM1uXDE9n8276hNdhohI4AI9IzjeNTS1tPcLxOLRNzbyh5XbKBwxhH+5Zhptw3KLiPRnoT4jeGNDFd95dg0AY3IG9dh+R+0BVpbXsHDFVppaNFS1iAwMoQ6C5tZWAH77pXNZMC2fjNSu+8ZPyh0Sz7K65e7UNjRxoLkl0aWIyAAS6iBoMzgtcoXslzedw4WTIzNwLph+8CHop75wTkLq6uyHL67j9O/+kTO//xI1+5t6foOISAxC3UfQ2YSRmTx6w1nty4/fcBbLP65OYEUH3fxECa+tjwyvsfdAM7N+8DK/vOkczhibk+DKRKS/C00QrN0WeS7u5ieWcd+VpzMmJ6PH98ydPJK5k0eys64BgNzMdG6cXcj9i9Ydcz3vb9nDj19aT3pKEsOHpFNeXc+MccO4cXYhKclJ/G55Bfe98CF1Bw4+0D1pVBbrdtQBsK+xhZXlexQEInLMQhMEb2yoinwvreL9LXtiCoI2uUPSAbjzksnsqG3o1ed+63erWLa5mvOKcvnWZVOAyG/0v1q6hdfWHTqA3usbqviXlzd0u69/uOwUUpKMv/lPjdgtIn1HfQQxSEoyNt17GVeeWdDr976wejsfbq9j0ZodALxZWsXUuxfx5Lsf92o/v/z7s7lgYh6zJ+RS8u2Lel2HiEh3FAS99MqHOwGYde8rlO6s67bdztoGZnz/T+zuMGzFvgPNXP/owd/mH7j6DAB+cMVpzBw//JD3n3niMDb+4FI23XsZm+69jFkTchERCUJoLg31lQ+ifQ1Vew9QVrmPCSOzDmvz4uptfPG/3wNg8ugsPtxex8e76zn17kXtbfJzMrhiRgFXzIicZVw78+CU0M+8V87FU0Z1+8Baa3SY1O88u4Z3Nu7mC+efzGkF2X1zgCISOqE9I3inbBf/+frGY95PS6uzqWofu/YeHLp6/Y697a/nTBp5SPvczDRe+Mp5vHnnvG73ecWMArIGpXa7vXrfwVtH/7ByG9f+dMnRlC4iAoT4jOB3yytYvmUP08flkD8s9o7j9JQk6hsjD3T9w+9WUfVfywBIssh/+m2Xjszg+rNP5OufnMhDiz9qf/9D15/JKSccafTunp04YvAhyw1NesBMRI5eaIMAIr+d/+6W2b16zy9vOoc7fvU+H26vo2rvwev/rX6w/wAi1///anrkss/3Lj+V1zdUMWl0JsWFww/bZ28NSk1m072XsXtfIz9+aT1PvL2ZhqYWBqUm09LqlFfXs2HHXrIGpTDjxGGkJof2xE9EYhDqIDgap5wwlC9ccBJ3/GrFEdu1hQDA52YV8rlZhX1ey/AhaTy3chsAk7/zYpdtrp05liVlu9lYtY+cwak8e+tsThxxfAyZISLHBwXBUWhq7n7AuYtOGXlYv0CQahuOPNTEk+8eHF11T30TFXv2HxIEO+saqKlv4sQRQ0hL0ZmDSBgpCI7BxVNG8a/XTucrTy1n0ZodPHT9DOZPPSGuNSz66vm8sHo7D/xpPS2tkYB66855bK9t4Ip/fwuAv//EePY3tfCLdz7m4cVlLNtUzdaaBp5buZXahsiTy9ecNZZ7P3N6XGsXkeODguAoXHLaaP7vb1fyzUsmMyg1mdkTclm0ZgcTRx1+K2nQTsrL5Na5E7h17gS21ewnJyONjLRkRg2NDKv96y+cy8zxw3n0jcgdUovXV7J4feVh+3lq6RZOzsvkpvNPYnVFDZt31TNlzFDGHycjr4pIcMy9f42rX1xc7CUlvZ/f/szv/6l9TuKJozLbb/HcdO9lfVrf8eqt0iqu6zQ0xRkF2Tx72ycovPO5Lt8zfVxOrzvTReT4ZGbL3L24q22hPCPoeJ9/WMyakNtt6N04u5DH39x02PqGptaAqxKR40Eog0AOdfdfnsqVZxbw6oc7OTU/m7mTRnLTEyWUV+9PdGkiEgcKAgHg1DHZnDpGw1SIhJGCQLq0fkcdm3fVt/cfXH7GGL58YRETRmYmuDIR6WsKAunS5l31hywvXLGVhSu2cuaJw/jcrEKmj82hYFhGtwPjiUj/Eeog+OMd5ye6hOPWF84/iVUVNdw+r+iQQe2Wba5m2eaD03cOSUvmngVTyctKZ9e+A5x7Ui6jswclomQROUqhDoJE3PffX3zz0lPaX2+69zKefb+CX5ds4c3SXYe029fYwtefPjjcxmdmFPCj6DwLItI/hDoIJHYLpuWzYFp++/Kq8hpeXbeTB/60HoAZ43JYWV7DgWaNhCrS3ygI5KicVpDNaQXZ3DLnZMqr91OYO4TCO5/jDyu3MXP8Jj57bmGiSxSRGGmUMTkmKclJFHYahuKuZ9fwzHvl7WMficjxTUEgfebhvz2z/fXXfr2CNVtrEliNiMQq0CAws/lmts7MSs3szi62f83M1prZSjN72cxODLIeCdZfnDqax24oZlBq5MeqqUVnBCL9QWBBYGbJwIPAJcAU4Fozm9Kp2XKg2N1PB34D3BdUPRIf8yaP4p+uitw19Jn/eIurHnqLF1dvT3BVInIkQXYWzwRK3b0MwMyeAhYAa9sauPurHdovAa4PsB6Jk617Do5RtHRTNY3NpUwencXrGyr5/ftbaW5pZfq4YXz38lMTWKWItAkyCPKBLR2Wy4Gzj9D+88ALXW0ws5uBmwHGjRvXV/VJQG467yQyUpPJGZzG7U8uZ0V5DXP+6bVD2qwor+Hrn5xI1qDUHvdXXl3PjtoDTMjLJHtwz+1FpHeOi9tHzex6oBi4oKvt7v4I8AhE5iM41s+76JRRPHT9jGPdjXTDzPjb6O2jtz+5/JBtT3/xXK566G0ATvvuH1kwbQzjc4cwe0IuORmpvLpuJ6sravlgWy0bdh46XPgFE/P4+d/NjMsxiIRJkEFQAYztsFwQXXcIM7sI+BZwgbsfCLCedoNSk0hJ1g1T8dDVHAgd5z949v2tAPz4pQ3d7mPSqCzW7ahj8fpKVlfUMDVfo6SK9KUgg2ApUGRm44kEwDXAdR0bmNl04GFgvrvvDLAWOY7c/ZenctN5JzHr3le49LTRPL8q0pk8e8IINlXVc3XxWM45aTjjc4cwMjrlZtsoqJ/61zeYdfII7rvydAqGDU7YMYgMJIEFgbs3m9ltwCIgGXjM3deY2T1AibsvBO4HMoGno6NYfuzulwdVkxw/xuRk9Gqa0O9dfip3L1wDwFsf7eLJdz/m9nlFbKtpYEh6MiOzNNCdyNEK5ZzFnzr9BP7tOvUR9EdvfVTFdT99p9vtmekpZKQlU1l3+FXGmeOHc9enpujSkoSS5iyWAeOswuFH3L73QDN7DzR3ue3djbv51L++wSVTR3NVcQEFwwZTNDJTcypI6CkIpF9JTU467JJSeXU9SWY0NrdS39jC5l37OOekEWyvbWBEZhojswbR3NLKhG9F7k5+YfV2Xog+5PaLvz+b2RNy434cIscTBYH0e507jaeMGQrAsCFp7etSogHyb69s4OHFZdRFzxrue/FDfnzNdFKTjfU76nh3YzUbdtQxKDWZc08ewbzJI0lNTuJAcwu5mekMSk2O34GJxImCQELltnlF3DaviJJNu7nyobdZUV7D3E4Pu7V5btW2bvfz+I1nMXfSyICqFIkv3UwvoVRcOJxHOoyWOndSHgXDMnjmlln84fZPcMdFE4/4/hsfX8p/vl7Gm6VVrNteR3+76UKko1CeEcQyrIEMfJ88dXS3t7BOzc/mKxcVHba+vrGZKXctAuAfn/ugfX1achKTRmdRs7+JiaMySU9N5r3N1VTtPUBqchLXzRzH35xzIoUjBqtzWo47obt99NLTRvPA1dN0rVeOWmur85NXNvD4m5uo2d/Uq/fOnjCCq4vHkjUohan52QwfnKan3CUudPtoB/csmKoQkGOSlGR89aKJfDV6+Wjppt2cXpBN1d5G1m+vo6xqH5efMYbczDSaWpzd+xo55wcvA/Bm6S7eLN11yP6m5g/lljkTGJ87hMmjs3TGIHEXuiAQ6Wttzzbk52SQn5PB3A7b0lKM0dmD2HTvZfzghQ9YU1HLCdmDeHpZeXub1RW13PKL9wAYnzuEr15UxOkFOYzMSmdIuv6JSvD0UyYSJ9+85JT21/dHJ+/5/fIKnn2/gmWbq6ltaGZj1T6+8tT77e3M4JzxIzh55BDmTR7JvMmj4l63DHwKApEE+vT0fD49Pb99+QcvfMBvl1VQtTcyRIY7vF22i7fLdvHfSz5ubzdiSBoHmlvZe6CZOZPyuOasseTnDKbFnTHZg9oH6xOJhYJA5DjyzUtOOeTM4fUNlQxJT2FJ2S7ue3Fd+/q2cbMAXltXyWvrKrvd59T8oayuqCXJ4NPT8ikuHE5DUwu5WenkZKTS1NJKS6uTnppMXUMT9Y0t7DvQzOkFOcwYl6M+ixBQEIgcx84rygNgxrhh3DJnAgANTS2kJieRnBT5D/qzj71LXUMTTS2trK6oPWwfbetaHZ5ZXsEzyw+bFqRbl0wdzbc/NYX8nIxjPRQ5jikIRPqZzne9PRHjrG2vrdvJ6xuqSEtJYtue/TS3Os0tTm5WGqvKaxiclsL0cTkMSU/hxdXbWVVRwwurt7Oxah8//WwxY4dr/oeBSkEgEhJzJo1kTozDYtw6dwL1jc2c98NX+XB7Hefd9yoXTMyjqaWVxuZW6hqaOa0gGwNaWp2zTxpO0ags8jLTFRj9kIJARLo0OC2FRXecT/E/vgRAzf4mUpON9z6uZmTWIN4qrQJga03DIZeb7r/ydLIGpXJW4TBGZKYnpHbpHQWBiHQrNzO9x5nkSnfWUV69n4Xvb+WZ5RV84zcrD9l+ct4QrphRQF5mOjmDU9lW08C44YM50NxKY0srI4akMW1sziHPTLg77pGH9yR4CgIROSYTRmYxYWQWF0zM4/YLi2hsbuWlD3aweF0lZVX7+KhyH/cvWtfzjoCM1GSaW1tpajk49M3gtGRmjh/Op6fl09LqtHrkKzsjlfOK8vTQXR/Qn6CI9AkzY3zuEAAmjc7i1rmRu5xq6puo3HuAhqYWqusbefujXVw8ZRRpKUlU72vi9dJKHl5cxl8Xj2VoRgopyUkkGazdWsvi9ZXUN7Yc8RbZ9JQk8odlMHbYYE7Oy+SCSXlkpiczafRQMhUSMdGfkogEKntwKtmDD47423ZLbJtPFOUe8uxEZ3vqG1lZXsPY4YNJMkgyo7ahiT+vr+LFNdtZt72Wssp9lFXuY/H6Sh57c2P7e3Mz0/mnq07HidyCm52hkYe7oiAQkeNazuA0zp+Yd9j6U8dk86U5J7cvuzulO/dSXd/Eq+t28ujrG6nae4AbHl/a3ua8olySk4yUJCPJjJRkY/e+RmadfOh0pfWNLZxflAsWOePIzkgjMz2F0dkD84ltBYGIDAhmRtGoLABmjh/O1y6eyIotewB49I2N7NrbyL4DzbS0Os2tTkur8+H2OgCWlO0+bH8PLf6oy8/JzUynoamFT08fg2HRz4bq+iamj83h5Q93kJ+Twaihg2hpdVrcaWmJfK/d38z0cTmcUZBDSvLBjnAzOGFoBkMzUhLyJLeCQEQGpNTkJIqjI8O2fe9KS+uhc7K0urOqoobG5lYq6w5gBo3Nrbz90S4GpyXzx7U7aGoxnl+1HYiciVTXR+al+J8VW9v3YwbJZiQnRb7qG1sA+O175RzJ0EEpnDE2h+ToWUvb5bAkM647e1yXZ0fHSkEgIqGW3OkW1WSMGeOGHdbuihkFAHxvwdQu91OzvwkckpJgSFrKYbe+ujvbaxvaz1I6+nB7HSvLa6io3s+g1CTqGppxd1qdQ+6Uqm3o3URIsVIQiIj0gZ46os2ME7IzOCH78HGb5k89IaiyYqI58kREQk5BICIScgoCEZGQUxCIiIScgkBEJOQUBCIiIRdoEJjZfDNbZ2alZnZnF9vTzexX0e3vmFlhkPWIiMjhAgsCM0sGHgQuAaYA15rZlE7NPg9Uu/sE4J+BHwZVj4iIdC3IM4KZQKm7l7l7I/AUsKBTmwXAz6OvfwNcaAEPtKFpLkREDhVkEOQDWzosl0fXddnG3ZuBGmBE5x2Z2c1mVmJmJZWVXY9J3pMfXX0GWekpmjpPRKSTftFZ7O6PuHuxuxfn5R3dgEtzJo1k1ff+oo8rExHp/4IMggpgbIflgui6LtuYWQqQDewKsCYREekkyCBYChSZ2XgzSwOuARZ2arMQ+Fz09ZXAK+7uiIhI3AQ2+qi7N5vZbcAiIBl4zN3XmNk9QIm7LwQeBf7LzEqB3UTCQkRE4ijQYajd/Xng+U7r7urwugG4KsgaRETkyPpFZ7GIiARHQSAiEnIKAhGRkFMQiIiEnPW3uzXNrBLYfJRvzwWq+rCc/kDHHA465nA4lmM+0d27fCK33wXBsTCzEncvTnQd8aRjDgcdczgEdcy6NCQiEnIKAhGRkAtbEDyS6AISQMccDjrmcAjkmEPVRyAiIocL2xmBiIh0oiAQEQm5ARkEZjbfzNaZWamZ3dnF9nQz+1V0+ztmVhj/KvtWDMf8NTNba2YrzexlMzsxEXX2pZ6OuUO7z5iZm1m/v9UwlmM2s6ujf9drzOyX8a6xr8Xwsz3OzF41s+XRn+9LE1FnXzGzx8xsp5mt7ma7mdlPon8eK81sxjF/qLsPqC8iQ15/BJwEpAErgCmd2twCPBR9fQ3wq0TXHYdjngsMjr7+UhiOOdouC/gzsAQoTnTdcfh7LgKWA8OiyyMTXXccjvkR4EvR11OATYmu+xiP+XxgBrC6m+2XAi8QmYL9HOCdY/3MgXhGMBModfcyd28EngIWdGqzAPh59PVvgAvNrD/Pa9/jMbv7q+5eH11cQmTGuP4slr9ngO8DPwQa4llcQGI55puAB929GsDdd8a5xr4WyzE7MDT6OhvYGsf6+py7/5nI/CzdWQA84RFLgBwzO+FYPnMgBkE+sKXDcnl0XZdt3L0ZqAFGxKW6YMRyzB19nshvFP1Zj8ccPWUe6+7PxbOwAMXy9zwRmGhmb5rZEjObH7fqghHLMX8XuN7MyonMf3J7fEpLmN7+e+9RoBPTyPHHzK4HioELEl1LkMwsCXgAuCHBpcRbCpHLQ3OInPX92cxOc/c9Ca0qWNcCP3P3H5nZuURmPZzq7q2JLqy/GIhnBBXA2A7LBdF1XbYxsxQip5O74lJdMGI5ZszsIuBbwOXufiBOtQWlp2POAqYCr5nZJiLXUhf28w7jWP6ey4GF7t7k7huB9USCob+K5Zg/D/wawN3fBgYRGZxtoIrp33tvDMQgWAoUmdl4M0sj0hm8sFObhcDnoq+vBF7xaC9MP9XjMZvZdOBhIiHQ368bQw/H7O417p7r7oXuXkikX+Rydy9JTLl9Ipaf7d8TORvAzHKJXCoqi2eRfSyWY/4YuBDAzE4hEgSVca0yvhYCn612hs8AAAJESURBVI3ePXQOUOPu245lhwPu0pC7N5vZbcAiInccPObua8zsHqDE3RcCjxI5fSwl0ilzTeIqPnYxHvP9QCbwdLRf/GN3vzxhRR+jGI95QInxmBcBnzSztUAL8A1377dnuzEe89eBn5rZHUQ6jm/oz7/YmdmTRMI8N9rvcTeQCuDuDxHpB7kUKAXqgRuP+TP78Z+XiIj0gYF4aUhERHpBQSAiEnIKAhGRkFMQiIiEnIJARCTkFAQiXTCzFjN738xWm9n/mFlOH+9/U/Q+f8xsb1/uW6S3FAQiXdvv7tPcfSqRZ01uTXRBIkFREIj07G2ig3qZ2clm9qKZLTOz181scnT9KDP7nZmtiH7Niq7/fbTtGjO7OYHHINKtAfdksUhfMrNkIsMXPBpd9QjwRXffYGZnA/8OzAN+Aix297+Kvicz2v7v3H23mWUAS83st/35SV8ZmBQEIl3LMLP3iZwJfAD8ycwygVkcHKYDID36fR7wWQB3byEytDnAl83sr6KvxxIZAE5BIMcVBYFI1/a7+zQzG0xknJtbgZ8Be9x9Wiw7MLM5wEXAue5eb2avERkQTeS4oj4CkSOIzur2ZSIDm9UDG83sKmifO/aMaNOXiUwBipklm1k2keHNq6MhMJnIUNgixx0FgUgP3H05sJLIBCh/A3zezFYAazg4beJXgLlmtgpYRmTu3BeBFDP7ALiXyFDYIscdjT4qIhJyOiMQEQk5BYGISMgpCEREQk5BICIScgoCEZGQUxCIiIScgkBEJOT+Fzhyky4F11iSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_col = \"pred_Sale\"\n",
    "col = \"Sale\"\n",
    "hist_multi_plot(df, pred_col, col, bins=np.arange(0,1, 0.01))\n",
    "precision, recall, _ = precision_recall_curve(df[col], df[pred_col])\n",
    "pr_display = PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "average_precision = average_precision_score(df[col], df[pred_col])\n",
    "print(\"Average Precision: \", average_precision)\n",
    "strawman_ce = log_loss(df[col], np.ones(df[col].shape)*means[col])\n",
    "print(\"Strawman Crossentropy\", strawman_ce)\n",
    "bce = log_loss(df[col], df[pred_col])\n",
    "print(\"Binary Crossentropy\", bce)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:insider]",
   "language": "python",
   "name": "conda-env-insider-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}