datastroyer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# and then we go to mario draghi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "import datetime\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "from statsmodels.distributions.empirical_distribution import ECDF\n",
    "import math\n",
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import pickle as pk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = pd.read_csv('datasets/1976-2020-president.csv')\n",
    "\n",
    "years = list(set(ad['year']))\n",
    "states = list(set(ad['state']))\n",
    "states.sort()\n",
    "years.sort()\n",
    "\n",
    "dflist = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "states.remove('DISTRICT OF COLUMBIA')\n",
    "years.remove(1976)\n",
    "years.remove(1980)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### some function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def foofloat(x):\n",
    "    try:\n",
    "        return float(x)\n",
    "    except:\n",
    "        return np.NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bls_serial(series_id):\n",
    "  bsobj = bs(requests.post(\"https://data.bls.gov/pdq/SurveyOutputServlet\", {'series_id': series_id}).text)\n",
    "  return bsobj.findAll('table')[0].findAll('tr')[2].findAll('td')[0].text.strip().upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ecdf_inv(a, q): return sorted(a)[int(len(a) * q)]\n",
    "def count_elem(l): return {y:list(l).count(y) for y in set(l)}\n",
    "def dpc(a,b): return (a/b - 1)*100\n",
    "def alfa(x): return ''.join([c for c in x if c.isalpha() or c==' ']).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_percent(older, col_key, shift=0):\n",
    "    newer = []\n",
    "    for y in years:\n",
    "        for s in states:\n",
    "            try:\n",
    "                d = older.loc[y-shift, s, 'DEMOCRAT'][col_key]\n",
    "                r = older.loc[y-shift, s, 'REPUBLICAN'][col_key]\n",
    "                newer.append([int(y), s, 100*d/(d+r)]) # % share for each state\n",
    "            except KeyError as e:\n",
    "                pass\n",
    "    return pd.DataFrame(data=newer, columns=['year', 'state', col_key, ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_splatter(a, col_name): # from row-column to table-list\n",
    "    lst = []\n",
    "    for y in a.columns[2:]:\n",
    "        for index, row in a.iterrows():\n",
    "            lst.append([int(y), row['GeoName'].upper(), row[y], ])\n",
    "    \n",
    "    return pd.DataFrame(data=lst, columns=['year', 'state', col_name, ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def var42(df_res, old_colname, new_colname, jump=1):\n",
    "    new_df = []\n",
    "    \n",
    "    df_res = df_res.set_index(['year', 'state'])\n",
    "    for y in years:\n",
    "        for s in states:\n",
    "            try:\n",
    "                new_df.append([y,s,] + [df_res.at[(y-i,s), old_colname] for i in range(0, 4, jump)])\n",
    "            except KeyError:\n",
    "                pass\n",
    "\n",
    "    new_df = pd.DataFrame(data=new_df, columns = ['year', 'state',] + [f'{new_colname}_{i}' for i in range(4//jump, 0, -1)])\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beadiff(rowcol, colname, diff = True, shift=1):\n",
    "    difflist = []\n",
    "\n",
    "    for s in states:\n",
    "        tmp = rowcol.loc[(slice(None),s),:]\n",
    "        yyyy = list({i[0] for i in tmp.index})\n",
    "        yyyy.sort()\n",
    "        for y in yyyy[shift:]:\n",
    "            if diff:\n",
    "                q = dpc(rowcol.at[(y,s), colname], rowcol.at[(int(y)-shift,s),colname])\n",
    "            else:\n",
    "                q = rowcol.at[(y,s),colname]\n",
    "            difflist.append([y, s, q])\n",
    "            \n",
    "    return  pd.DataFrame(data=difflist, columns=['year', 'state', colname])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def whore(df, yy=years, ss=states):\n",
    "    df = df.set_index(['year','state'])\n",
    "    x = []\n",
    "    for s in ss:\n",
    "        for y in yy:\n",
    "            try:\n",
    "                df.loc[(y,s),:]\n",
    "            except:\n",
    "                x.append((y,s))\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sell(df, filename=None):\n",
    "    df = df.sort_values(by=['year', 'state'])\n",
    "    dflist.append(df)\n",
    "    if filename is not None:\n",
    "        df.to_csv(f'datasets-clean/{filename}.csv', index=False)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### y_hat aka presitdent popular vote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>y_votes_percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>38.736605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>30.944087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>32.883272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>38.772412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>41.775465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>55.154687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>59.925503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>30.201468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>50.319063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>27.519566</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  y_votes_percent\n",
       "0    1984        ALABAMA        38.736605\n",
       "1    1984         ALASKA        30.944087\n",
       "2    1984        ARIZONA        32.883272\n",
       "3    1984       ARKANSAS        38.772412\n",
       "4    1984     CALIFORNIA        41.775465\n",
       "..    ...            ...              ...\n",
       "495  2020       VIRGINIA        55.154687\n",
       "496  2020     WASHINGTON        59.925503\n",
       "497  2020  WEST VIRGINIA        30.201468\n",
       "498  2020      WISCONSIN        50.319063\n",
       "499  2020        WYOMING        27.519566\n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pop_vote = pd.read_csv('datasets/1976-2020-president.csv')\n",
    "pop_vote = pop_vote[['year', 'state', 'party_simplified', 'candidatevotes', ]] # drop useless columns\n",
    "pop_vote = pop_vote.groupby(['year', 'state', 'party_simplified', ]).sum() # drop multiple candidate\n",
    "pop_vote = to_percent(pop_vote, 'candidatevotes')\n",
    "pop_vote = pop_vote.rename(columns={\"candidatevotes\": \"y_votes_percent\", })\n",
    "pop_vote = pop_vote.sort_values(by=['year', 'state'])\n",
    "pop_vote.to_csv('datasets-clean/popular-vote-y.csv', index=False)\n",
    "dflist.append(pop_vote)\n",
    "pop_vote"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GDP and friends"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It was necessary to generate an adjusted series of state GDP because of a change in BEA’s estimation procedure from a Standard Industrial Classification (SIC) basis to a North American Industry Classification System (NAICS) basis in 1997.\n",
    "Data prior to 1997 were adjusted to avoid any erratic shifts in GDP that year.\n",
    "While the change to NAICS basis occurred in 1997, BEA also provides estimates under a SIC basis in that year.\n",
    "~~Our adjustment involved calculating the 1997 ratio of NAICS-based GDP to SIC-based GDP for each state, and multiplying it by SIC-based GDP in all years prior to 1997 to obtain our adjusted series of state-level GDP.~~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_nom_97 = to_splatter(pd.read_csv('datasets/gdp-nominal-63-97.csv'), 'gdp_nom')\n",
    "gdp_nom_12 = to_splatter(pd.read_csv('datasets/gdp-nominal-97-20.csv'), 'gdp_nom')\n",
    "gdp_real_97 = to_splatter(pd.read_csv('datasets/gdp-real-77-97-chain-97.csv'), 'gdp_real')\n",
    "gdp_real_12 = to_splatter(pd.read_csv('datasets/gdp-real-97-20-chain-12.csv'), 'gdp_real')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_nom_97 = gdp_nom_97.set_index(['year', 'state'])\n",
    "gdp_nom_12 = gdp_nom_12.set_index(['year', 'state'])\n",
    "gdp_real_97 = gdp_real_97.set_index(['year', 'state'])\n",
    "gdp_real_12 = gdp_real_12.set_index(['year', 'state'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# moving to midterms (retrocompatibility)\n",
    "k = 2\n",
    "sx = '_mt' if k-1 else ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### price indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_def_97 = (gdp_nom_97['gdp_nom'] / gdp_real_97['gdp_real']).to_frame('gdp_def').dropna()\n",
    "gdp_def_12 = (gdp_nom_12['gdp_nom'] / gdp_real_12['gdp_real']).to_frame('gdp_def').dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>fair_p_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>26.788107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>30.073594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>27.619112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>25.124853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>27.168417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>7.549968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>6.646364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>7.411864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>7.602507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>6.633573</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state   fair_p_1\n",
       "0    1984        ALABAMA  26.788107\n",
       "1    1984         ALASKA  30.073594\n",
       "2    1984        ARIZONA  27.619112\n",
       "3    1984       ARKANSAS  25.124853\n",
       "4    1984     CALIFORNIA  27.168417\n",
       "..    ...            ...        ...\n",
       "495  2020       VIRGINIA   7.549968\n",
       "496  2020     WASHINGTON   6.646364\n",
       "497  2020  WEST VIRGINIA   7.411864\n",
       "498  2020      WISCONSIN   7.602507\n",
       "499  2020        WYOMING   6.633573\n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# price index FAIR\n",
    "fair_p = pd.concat([beadiff(gdp_def_12,'gdp_def',shift=4), beadiff(gdp_def_97,'gdp_def',shift=4)])\n",
    "\n",
    "# fix 97jump\n",
    "tmp = dpc(gdp_def_12.loc[2000], gdp_def_97.loc[1996])\n",
    "tmp['year'] = 2000\n",
    "fair_p = pd.concat([fair_p, tmp.reset_index(), ])\n",
    "\n",
    "# recolumning + appending\n",
    "fair_p = var42(fair_p, new_colname='fair_p', old_colname='gdp_def', jump=4)\n",
    "\n",
    "sell(fair_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>def_mt_2</th>\n",
       "      <th>def_mt_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>8.923985</td>\n",
       "      <td>16.400541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>0.219513</td>\n",
       "      <td>29.788690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>10.086238</td>\n",
       "      <td>15.926491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>8.706949</td>\n",
       "      <td>15.102903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>9.427608</td>\n",
       "      <td>16.212370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>4.216321</td>\n",
       "      <td>3.198776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>3.584248</td>\n",
       "      <td>2.956160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>0.648566</td>\n",
       "      <td>6.719717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>4.323340</td>\n",
       "      <td>3.143273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>-2.496421</td>\n",
       "      <td>9.363752</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state   def_mt_2   def_mt_1\n",
       "0    1984        ALABAMA   8.923985  16.400541\n",
       "1    1984         ALASKA   0.219513  29.788690\n",
       "2    1984        ARIZONA  10.086238  15.926491\n",
       "3    1984       ARKANSAS   8.706949  15.102903\n",
       "4    1984     CALIFORNIA   9.427608  16.212370\n",
       "..    ...            ...        ...        ...\n",
       "495  2020       VIRGINIA   4.216321   3.198776\n",
       "496  2020     WASHINGTON   3.584248   2.956160\n",
       "497  2020  WEST VIRGINIA   0.648566   6.719717\n",
       "498  2020      WISCONSIN   4.323340   3.143273\n",
       "499  2020        WYOMING  -2.496421   9.363752\n",
       "\n",
       "[500 rows x 4 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# OUR price index\n",
    "def_index = pd.concat([beadiff(gdp_def_12,'gdp_def', shift=k), beadiff(gdp_def_97,'gdp_def',shift=k)])\n",
    "\n",
    "if k-1:\n",
    "    tmp = dpc(gdp_def_12.loc[1996+k], gdp_def_97.loc[1996])\n",
    "    tmp['year'] = 1996+k\n",
    "    def_index = pd.concat([def_index, tmp.reset_index(), ])\n",
    "\n",
    "def_index = var42(def_index, new_colname='def'+sx, old_colname='gdp_def', jump=k)\n",
    "\n",
    "sell(def_index, 'price-deflator')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### population denom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pop = to_splatter(pd.read_csv('datasets/population-75-21.csv'), col_name='gdp_real')\n",
    "pop.state = pop.state.apply(alfa)\n",
    "pop = pop.set_index(['year', 'state'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### gdp indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# re-generate plain series (see despacito)\n",
    "gdp_real_plain = gdp_real_12.copy()\n",
    "for y in range(1996, 1977, -1):\n",
    "    for s in sorted(states):\n",
    "        a = gdp_real_97.loc[(y,s),'gdp_real']\n",
    "        b = gdp_real_97.loc[(y+1,s),'gdp_real']\n",
    "        c = gdp_real_plain.loc[(y+1,s),'gdp_real']\n",
    "\n",
    "        gdp_real_plain.at[(y,s),'gdp_real'] = a*b/c\n",
    "\n",
    "gdp_real_plain = gdp_real_plain.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>gdp_mt_2</th>\n",
       "      <th>gdp_mt_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>9.896919</td>\n",
       "      <td>-0.693998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>1.171316</td>\n",
       "      <td>18.284477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>17.082207</td>\n",
       "      <td>0.363747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>11.296819</td>\n",
       "      <td>0.810684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>11.894211</td>\n",
       "      <td>3.321546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>-0.837617</td>\n",
       "      <td>3.881593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>3.341932</td>\n",
       "      <td>12.518139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>-4.653816</td>\n",
       "      <td>4.431850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>-2.571740</td>\n",
       "      <td>2.567721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>-4.557241</td>\n",
       "      <td>-0.525810</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state   gdp_mt_2   gdp_mt_1\n",
       "0    1984        ALABAMA   9.896919  -0.693998\n",
       "1    1984         ALASKA   1.171316  18.284477\n",
       "2    1984        ARIZONA  17.082207   0.363747\n",
       "3    1984       ARKANSAS  11.296819   0.810684\n",
       "4    1984     CALIFORNIA  11.894211   3.321546\n",
       "..    ...            ...        ...        ...\n",
       "495  2020       VIRGINIA  -0.837617   3.881593\n",
       "496  2020     WASHINGTON   3.341932  12.518139\n",
       "497  2020  WEST VIRGINIA  -4.653816   4.431850\n",
       "498  2020      WISCONSIN  -2.571740   2.567721\n",
       "499  2020        WYOMING  -4.557241  -0.525810\n",
       "\n",
       "[500 rows x 4 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# OUR two-lag + fix 97jump\n",
    "gdp_real_var = pd.concat([beadiff(gdp_real_97,'gdp_real', shift=k), beadiff(gdp_real_12,'gdp_real', shift=k)])\n",
    "\n",
    "if k-1:\n",
    "    tmp = gdp_real_plain.set_index(['year','state'])\n",
    "    tmp = dpc(tmp.loc[1996+k],tmp.loc[1996])\n",
    "    tmp['year'] = 1996+k\n",
    "    gdp_real_var = pd.concat([gdp_real_var, tmp.reset_index(), ])\n",
    "    \n",
    "gdp_index = var42(gdp_real_var, new_colname='gdp'+sx, old_colname='gdp_real', jump=k)\n",
    "\n",
    "sell(gdp_index, 'gdp-growth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "if True:\n",
    "    gdp_real_12 /= pop.loc[gdp_real_12.index]\n",
    "    gdp_real_97 /= pop.loc[gdp_real_97.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>fair_g_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>4.249743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>-1.123909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>7.206264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>7.188213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>6.003924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>-3.141094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>-1.610299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>-3.338880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>-4.139381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>-6.496316</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  fair_g_1\n",
       "0    1984        ALABAMA  4.249743\n",
       "1    1984         ALASKA -1.123909\n",
       "2    1984        ARIZONA  7.206264\n",
       "3    1984       ARKANSAS  7.188213\n",
       "4    1984     CALIFORNIA  6.003924\n",
       "..    ...            ...       ...\n",
       "495  2020       VIRGINIA -3.141094\n",
       "496  2020     WASHINGTON -1.610299\n",
       "497  2020  WEST VIRGINIA -3.338880\n",
       "498  2020      WISCONSIN -4.139381\n",
       "499  2020        WYOMING -6.496316\n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one-lag FAIR\n",
    "fair_g = pd.concat([beadiff(gdp_real_97,'gdp_real'), beadiff(gdp_real_12,'gdp_real')])\n",
    "fair_g = var42(fair_g, new_colname='fair_g', old_colname='gdp_real', jump=4)\n",
    "\n",
    "sell(fair_g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "if True:\n",
    "    gdp_real_12 *= pop.loc[gdp_real_12.index]\n",
    "    gdp_real_97 *= pop.loc[gdp_real_97.index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### z growth index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.2"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = pd.concat([beadiff(gdp_real_97,'gdp_real'), beadiff(gdp_real_12,'gdp_real')])\n",
    "quantile = ECDF(pd.read_csv('datasets/us-gdp-quarter-47-21-varpc.csv').GDP_PCH)(3.2)\n",
    "\n",
    "'''for s in states:\n",
    "    l = tmp[tmp.state == s].gdp_real\n",
    "    print(f'{s.ljust(max([len(i) for i in states]))} {ecdf_inv(l, quantile):10.2f}', )'''\n",
    "\n",
    "GDP_THRESHOLD = sorted([ecdf_inv(tmp[tmp.state == s].gdp_real, quantile) for s in states])[0] if False else 3.2\n",
    "GDP_THRESHOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>z_4</th>\n",
       "      <th>z_3</th>\n",
       "      <th>z_2</th>\n",
       "      <th>z_1</th>\n",
       "      <th>z_mt_2</th>\n",
       "      <th>z_mt_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  z_4  z_3  z_2  z_1  z_mt_2  z_mt_1\n",
       "0    1984        ALABAMA    1    1    0    0       1       0\n",
       "1    1984         ALASKA    1    0    0    1       1       1\n",
       "2    1984        ARIZONA    1    1    0    0       1       0\n",
       "3    1984       ARKANSAS    1    1    0    1       1       1\n",
       "4    1984     CALIFORNIA    1    1    0    1       1       1\n",
       "..    ...            ...  ...  ...  ...  ...     ...     ...\n",
       "495  2020       VIRGINIA    0    0    0    0       0       0\n",
       "496  2020     WASHINGTON    0    1    1    1       1       1\n",
       "497  2020  WEST VIRGINIA    0    0    0    0       0       0\n",
       "498  2020      WISCONSIN    0    0    0    0       0       0\n",
       "499  2020        WYOMING    0    0    0    0       0       0\n",
       "\n",
       "[500 rows x 8 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zk = 1\n",
    "zsx = '_mt' if zk-1 else ''\n",
    "growth_index = pd.concat([beadiff(gdp_real_97,'gdp_real', shift=zk), beadiff(gdp_real_12,'gdp_real', shift=zk)]) ## NON CONVINTO, brick on 97\n",
    "growth_index.gdp_real = growth_index.gdp_real.apply(lambda x : 1 if x > GDP_THRESHOLD else 0)\n",
    "growth_index = var42(growth_index, new_colname='z'+zsx, old_colname='gdp_real', jump=zk)\n",
    "\n",
    "growth_index['z_mt_2'] = (growth_index.z_4 + growth_index.z_3).apply( lambda x: 1 if x else 0)\n",
    "growth_index['z_mt_1'] = (growth_index.z_2 + growth_index.z_1).apply( lambda x: 1 if x else 0)\n",
    "\n",
    "sell(growth_index, 'z-growth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### incumbent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>incumbent</th>\n",
       "      <th>former_president_again</th>\n",
       "      <th>former_party_morethan_2</th>\n",
       "      <th>sudden_vice</th>\n",
       "      <th>lag_vice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  incumbent  former_president_again  \\\n",
       "0    1984        ALABAMA         -1                      -1   \n",
       "1    1984         ALASKA         -1                      -1   \n",
       "2    1984        ARIZONA         -1                      -1   \n",
       "3    1984       ARKANSAS         -1                      -1   \n",
       "4    1984     CALIFORNIA         -1                      -1   \n",
       "..    ...            ...        ...                     ...   \n",
       "495  2020       VIRGINIA         -1                      -1   \n",
       "496  2020     WASHINGTON         -1                      -1   \n",
       "497  2020  WEST VIRGINIA         -1                      -1   \n",
       "498  2020      WISCONSIN         -1                      -1   \n",
       "499  2020        WYOMING         -1                      -1   \n",
       "\n",
       "     former_party_morethan_2  sudden_vice  lag_vice  \n",
       "0                        0.0            0         0  \n",
       "1                        0.0            0         0  \n",
       "2                        0.0            0         0  \n",
       "3                        0.0            0         0  \n",
       "4                        0.0            0         0  \n",
       "..                       ...          ...       ...  \n",
       "495                      0.0            0         1  \n",
       "496                      0.0            0         1  \n",
       "497                      0.0            0         1  \n",
       "498                      0.0            0         1  \n",
       "499                      0.0            0         1  \n",
       "\n",
       "[500 rows x 7 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incumbent = pd.read_csv('datasets/incumbent-4president-76-20.csv', sep=';')\n",
    "tmp = pd.DataFrame(data=[[y,s] for y in years for s in states], columns=['year', 'state'])\n",
    "incumbent = pd.merge(tmp, incumbent, how='inner', left_on='year', right_on='year')\n",
    "\n",
    "sell(incumbent, 'incumbent-longrep')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### house dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>house_midterm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  house_midterm\n",
       "0    1984        ALABAMA              1\n",
       "1    1984         ALASKA             -1\n",
       "2    1984        ARIZONA             -1\n",
       "3    1984       ARKANSAS              1\n",
       "4    1984     CALIFORNIA              1\n",
       "..    ...            ...            ...\n",
       "484  2020       VIRGINIA              1\n",
       "485  2020     WASHINGTON              1\n",
       "486  2020  WEST VIRGINIA             -1\n",
       "487  2020      WISCONSIN              1\n",
       "488  2020        WYOMING             -1\n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "house_vote = pd.read_csv('datasets/1976-2020-house-utf8.csv')\n",
    "house_vote = house_vote[['year', 'state', 'party', 'candidatevotes', ]]\n",
    "house_vote = house_vote.groupby(['year', 'state', 'party', ]).sum()\n",
    "house_vote = to_percent(house_vote, 'candidatevotes', shift=2)\n",
    "house_vote['candidatevotes'] = house_vote['candidatevotes'].apply(lambda x: -1 if x < 50 else 1)\n",
    "house_vote = house_vote.rename(columns={'candidatevotes':'house_midterm'})\n",
    "house_vote = pd.concat([house_vote, pd.read_csv('datasets/midterm-fixed.csv')])\n",
    "house_vote.state = house_vote.state.apply(lambda x : x.upper())\n",
    "\n",
    "sell(house_vote, 'midterm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### personal income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>avg_inc_mt_2</th>\n",
       "      <th>avg_inc_mt_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>16.997412</td>\n",
       "      <td>16.526329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>1.426071</td>\n",
       "      <td>25.259560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>17.224666</td>\n",
       "      <td>15.200083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>16.832890</td>\n",
       "      <td>18.783730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>15.239614</td>\n",
       "      <td>15.470908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>8.740216</td>\n",
       "      <td>6.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>11.224152</td>\n",
       "      <td>9.655147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>8.665411</td>\n",
       "      <td>10.554562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>8.474146</td>\n",
       "      <td>7.874298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>3.246536</td>\n",
       "      <td>9.882249</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  avg_inc_mt_2  avg_inc_mt_1\n",
       "0    1984        ALABAMA     16.997412     16.526329\n",
       "1    1984         ALASKA      1.426071     25.259560\n",
       "2    1984        ARIZONA     17.224666     15.200083\n",
       "3    1984       ARKANSAS     16.832890     18.783730\n",
       "4    1984     CALIFORNIA     15.239614     15.470908\n",
       "..    ...            ...           ...           ...\n",
       "495  2020       VIRGINIA      8.740216      6.622504\n",
       "496  2020     WASHINGTON     11.224152      9.655147\n",
       "497  2020  WEST VIRGINIA      8.665411     10.554562\n",
       "498  2020      WISCONSIN      8.474146      7.874298\n",
       "499  2020        WYOMING      3.246536      9.882249\n",
       "\n",
       "[500 rows x 4 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inc_cap = to_splatter(pd.read_csv('datasets/personal-income-per-capita-72-20.csv'), 'avg_inc')\n",
    "inc_cap = inc_cap.set_index(['year', 'state'])\n",
    "#inc_cap.avg_inc = inc_cap.avg_inc.apply(lambda x : math.log(x))\n",
    "inc_cap = beadiff(inc_cap, 'avg_inc', shift=k)\n",
    "inc_cap = var42(inc_cap,  'avg_inc', 'avg_inc'+sx, jump=k)\n",
    "\n",
    "sell(inc_cap, 'income')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### unenployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "unn = pd.read_csv('datasets/unemployment-76-21-percent.csv')\n",
    "furi = 'datasets-clean/serial-id-bsl.pkl'\n",
    "\n",
    "if False:\n",
    "    sd = dict()\n",
    "    for s in set([*unn['Series ID'], *nunn['Series ID']]):\n",
    "        try:\n",
    "            sd[s] = bls_serial(s)\n",
    "        except IndexError:\n",
    "            sd[s] = f'BADASS_{s}'\n",
    "    pk.dump(sd, open(furi, 'wb'))\n",
    "else:\n",
    "    sd = pk.load(open(furi, 'rb'))\n",
    "\n",
    "unn = unn.rename(columns={\n",
    "    \"Series ID\": \"state\",\n",
    "    \"Year\": \"year\",\n",
    "    \"Value\": \"unemp\",\n",
    "    \"Period\": \"month\",\n",
    "    })\n",
    "\n",
    "# transform\n",
    "unn.unemp = unn.unemp.apply(foofloat).astype(float)\n",
    "unn.state = unn.state.map(sd)\n",
    "unn.month = unn.month.apply(lambda x: x.split('M')[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace missing value\n",
    "glue = pd.read_csv('datasets/unemployment-rate-us-country-47-21.csv')\n",
    "glue['year'] = glue.DATE.apply(lambda x: x.split('-')[0])\n",
    "glue['month'] = glue.DATE.apply(lambda x: x.split('-')[1])\n",
    "glue = glue.set_index(['year','month'])\n",
    "\n",
    "for i in unn.index:\n",
    "    unn.at[i, \"unemp\"] = glue.at[(unn.at[i, \"year\"], unn.at[i, \"month\"]), 'UNRATE'] if unn.at[i, \"unemp\"] == np.NaN else unn.at[i, \"unemp\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>unemp_rn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>11.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1984</td>\n",
       "      <td>ALASKA</td>\n",
       "      <td>9.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARIZONA</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1984</td>\n",
       "      <td>ARKANSAS</td>\n",
       "      <td>8.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>CALIFORNIA</td>\n",
       "      <td>7.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2020</td>\n",
       "      <td>VIRGINIA</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>2020</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>9.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2020</td>\n",
       "      <td>WEST VIRGINIA</td>\n",
       "      <td>8.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>2020</td>\n",
       "      <td>WISCONSIN</td>\n",
       "      <td>6.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>6.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year          state  unemp_rn\n",
       "0    1984        ALABAMA      11.1\n",
       "1    1984         ALASKA       9.8\n",
       "2    1984        ARIZONA       4.5\n",
       "3    1984       ARKANSAS       8.4\n",
       "4    1984     CALIFORNIA       7.6\n",
       "..    ...            ...       ...\n",
       "495  2020       VIRGINIA       7.0\n",
       "496  2020     WASHINGTON       9.1\n",
       "497  2020  WEST VIRGINIA       8.5\n",
       "498  2020      WISCONSIN       6.1\n",
       "499  2020        WYOMING       6.2\n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# delta%\n",
    "dmp = unn.set_index(['year','month','state'])\n",
    "\n",
    "unemp_mt = []\n",
    "unemp_s = []\n",
    "for y in years:\n",
    "    for s in states:\n",
    "        try:\n",
    "            m2 = dpc(dmp.at[(y, '10', s), 'unemp'], dmp.at[(y-2, '11', s), 'unemp'])\n",
    "            m1 = dpc(dmp.at[(y-2, '10', s), 'unemp'], dmp.at[(y-3, '01', s), 'unemp'])\n",
    "            unemp_mt.append([y,s,m1,m2,])\n",
    "\n",
    "            unemp_s.append([y, s, dmp.loc[[(y, f'{i:02}', s) for i in range(6,11)], 'unemp'].median(), ])\n",
    "        except KeyError:\n",
    "            pass\n",
    "        \n",
    "unemp_mt = pd.DataFrame(data=unemp_mt, columns=['year', 'state', 'unemp_mt_1', 'unemp_mt_2'])\n",
    "unemp_s = pd.DataFrame(data=unemp_s, columns=['year', 'state', 'unemp_rn',])\n",
    "\n",
    "sell(unemp_s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merging features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=[[y,s] for y in years for s in states], columns=['year', 'state'])\n",
    "for d in dflist:\n",
    "    df = df.merge(d, how='inner', left_on=['year','state'], right_on=['year','state'])\n",
    "\n",
    "df = df.sort_values(by=['year', 'state'])\n",
    "\n",
    "df.to_csv('datasets-clean/xxx-2012-insample-final-dataset.csv', index=False)\n",
    "df[df.year != 2020].to_csv('datasets-clean/xxx-2020-outofsample-final-dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in dflist:\n",
    "    w = whore(d)\n",
    "    if len(w):\n",
    "        print(d.head())\n",
    "        print('-+'*30)\n",
    "        print(w)\n",
    "        print('-+'*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_200/1320946542.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;36m0\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# chernobyl zone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "0/0\n",
    "# chernobyl zone"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "exp fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = []\n",
    "for y in years:\n",
    "    for s in states:\n",
    "        m2 = unn[unn.year >= y-1][unn.year <= y][unn.state == s].unemp.median()\n",
    "        m1 = unn[unn.year >= y-3][unn.year <= y-2][unn.state == s].unemp.median()\n",
    "        tmp.append([y,s,m1,m2])\n",
    "mid_unem = pd.DataFrame(data=tmp, columns=['year', 'state', 'unemp_12', 'unemp_34'])\n",
    "dflist.append(mid_unem)\n",
    "mid_unem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unn = var42(unn, 'unemp', 'unemployment-index', 'unemp')\n",
    "dflist.append(unn)\n",
    "unn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_index['gdp_34'] = (gdp_index['gdp_3']+gdp_index['gdp_3'])/2\n",
    "gdp_index['gdp_12'] = (gdp_index['gdp_1']+gdp_index['gdp_2'])/2\n",
    "\n",
    "del gdp_index['gdp_1']\n",
    "del gdp_index['gdp_2']\n",
    "del gdp_index['gdp_3']\n",
    "del gdp_index['gdp_4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_boot = gdp_real.copy()\n",
    "gdp_exp = gdp_boot.set_index(['year', 'state'])\n",
    "for s in states:\n",
    "    print(gdp_exp.loc[(slice(None),s),:].gdp_real.apply(lambda x : math.exp((x - gdp_exp.gdp_real.mean())/ math.sqrt(gdp_exp.gdp_real.var()))))\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "us stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_us = pd.read_csv('datasets/gdp-nomina-47-20-chain-12.csv')\n",
    "gdp_us.DATE = gdp_us.DATE.apply(lambda x: int(str(x).split('-')[0]))\n",
    "s = gdp_us[gdp_us.DATE > 1975].GDPC1_PC1\n",
    "s.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecdf = ECDF(s)\n",
    "ecdf(2.8)\n",
    "\n",
    "qnt = 0.67\n",
    "std_err = np.sqrt(s.var())/2\n",
    "ecdf_inv(s, qnt), len(s)*(1-qnt), std_err"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### house"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "house_vote = pd.read_csv('datasets/1976-2020-house-utf8.csv')\n",
    "house_vote = house_vote[['year', 'state', 'party', 'candidatevotes', ]]\n",
    "house_vote = house_vote.groupby(['year', 'state', 'party', ]).sum()\n",
    "house_vote = to_percent(house_vote, 'candidatevotes')\n",
    "house_vote = house_vote.rename(columns={\"candidatevotes\": \"houserep_votes_percent\", })\n",
    "#'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### gasoline and friends + approval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas = pd.read_csv('datasets/gasoline-93-21.csv')\n",
    "gas.date = gas.date.apply(lambda x: time.mktime(datetime.datetime.strptime(x,\"%m/%d/%Y\").timetuple()))\n",
    "gas.date = gas.date.apply(lambda x: datetime.datetime.fromtimestamp(int(x)))\n",
    "gas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### houses prices and rent + personal income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('datasets/house-chain-00.csv', 'rt')\n",
    "lines = f.readlines()\n",
    "houses = [[i.strip() for i in l.split(sep = '$') ] for l in lines ]\n",
    "houses = pd.DataFrame(data=houses, columns=['state', 2000, 1990, 1980, 1970, 1960, 1950, 1940 ])\n",
    "houses.head()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}