word_timeseries.ipynb

{
 "metadata": {
  "name": "word_timeseries"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "from pandas import DataFrame\n",
      "from pandas import Series\n",
      "from datetime import datetime\n",
      "MAX_YEAR = datetime.now().year"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "import codecs\n",
      "import json\n",
      "words_ts = []\n",
      "words_stats = []\n",
      "words_dict = {'word': [], 'total': [], 'average': [], 'year_counts': [], 'num_years': []}\n",
      "with codecs.open('ngrams/timeseries_data/all', encoding='utf-8') as f:\n",
      "    for line in f:\n",
      "        m = re.match(r'\\\"([\\w\\\\]+)\\\"\\s([\\[\\]\\w\\s\\,]+)', line)\n",
      "        if m:\n",
      "            key = m.group(1)\n",
      "            value = json.loads(m.group(2))\n",
      "            maximum = value[0]\n",
      "            total = value[1]\n",
      "            num_years = value[2]\n",
      "            time_tuples = map(tuple, value[3])\n",
      "            #Create List of (Word, total count, average count, maximum) tuples\n",
      "            #words_stats.append( (key, total, float(total)/num_years, maximum, num_years, time_tuples) )\n",
      "            #Create list (Word, list((year, count))\n",
      "            #words_ts.append( (key, time_tuples) )\n",
      "            words_dict['word'].append(key)\n",
      "            words_dict['total'].append(int(total))\n",
      "            words_dict['average'].append(float(total)/num_years)\n",
      "            words_dict['year_counts'].append(time_tuples)\n",
      "            words_dict['num_years'].append(num_years)\n",
      "        else:\n",
      "            print \"no match\", line"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Construct a Pandas DataFrame"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words_df = DataFrame(words_dict)\n",
      "#Copy the word column to use one was index and preserve the other\n",
      "words_df['key'] = words_df['word']\n",
      "words_df = words_df.set_index('key')\n",
      "words_df.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>average</th>\n",
        "      <th>num_years</th>\n",
        "      <th>total</th>\n",
        "      <th>word</th>\n",
        "      <th>year_counts</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>key</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0001</th>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0001</td>\n",
        "      <td>                                       [(1997, 1)]</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0012</th>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0012</td>\n",
        "      <td>                                       [(2010, 1)]</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>003</th>\n",
        "      <td> 1</td>\n",
        "      <td> 5</td>\n",
        "      <td> 5</td>\n",
        "      <td>  003</td>\n",
        "      <td> [(2002, 1), (1980, 1), (1945, 1), (1969, 1), (...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0056</th>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0056</td>\n",
        "      <td>                                       [(1987, 1)]</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0081</th>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0081</td>\n",
        "      <td>                                       [(2009, 1)]</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "      average  num_years  total  word  \\\n",
        "key                                     \n",
        "0001        1          1      1  0001   \n",
        "0012        1          1      1  0012   \n",
        "003         1          5      5   003   \n",
        "0056        1          1      1  0056   \n",
        "0081        1          1      1  0081   \n",
        "\n",
        "                                            year_counts  \n",
        "key                                                      \n",
        "0001                                        [(1997, 1)]  \n",
        "0012                                        [(2010, 1)]  \n",
        "003   [(2002, 1), (1980, 1), (1945, 1), (1969, 1), (...  \n",
        "0056                                        [(1987, 1)]  \n",
        "0081                                        [(2009, 1)]  "
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Cleaning up the (year, count) tuples and turning them into a Pandas.Series\n",
      "---------------------------------------------------------------------------"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The 'None' appears to be a problem here\n",
      "#Also the last couple of entries are noise in the data, as they lie in the future\n",
      "sorted(words_df.ix['new']['year_counts'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "[(None, 175),\n",
        " (1538, 1),\n",
        " (1549, 1),\n",
        " (1550, 1),\n",
        " (1551, 1),\n",
        " (1565, 1),\n",
        " (1570, 1),\n",
        " (1577, 2),\n",
        " (1580, 1),\n",
        " (1581, 1),\n",
        " (1582, 1),\n",
        " (1585, 1),\n",
        " (1586, 1),\n",
        " (1590, 1),\n",
        " (1593, 1),\n",
        " (1594, 3),\n",
        " (1595, 3),\n",
        " (1596, 6),\n",
        " (1598, 1),\n",
        " (1599, 2),\n",
        " (1600, 2),\n",
        " (1601, 1),\n",
        " (1602, 2),\n",
        " (1603, 2),\n",
        " (1604, 2),\n",
        " (1605, 2),\n",
        " (1606, 1),\n",
        " (1607, 3),\n",
        " (1608, 2),\n",
        " (1609, 5),\n",
        " (1610, 2),\n",
        " (1611, 1),\n",
        " (1612, 3),\n",
        " (1613, 6),\n",
        " (1614, 1),\n",
        " (1615, 2),\n",
        " (1617, 4),\n",
        " (1618, 3),\n",
        " (1619, 3),\n",
        " (1620, 12),\n",
        " (1621, 1),\n",
        " (1622, 3),\n",
        " (1623, 1),\n",
        " (1624, 6),\n",
        " (1625, 2),\n",
        " (1626, 5),\n",
        " (1627, 1),\n",
        " (1628, 8),\n",
        " (1629, 3),\n",
        " (1630, 5),\n",
        " (1631, 6),\n",
        " (1632, 2),\n",
        " (1633, 6),\n",
        " (1634, 6),\n",
        " (1635, 1),\n",
        " (1636, 1),\n",
        " (1637, 3),\n",
        " (1638, 6),\n",
        " (1639, 2),\n",
        " (1640, 2),\n",
        " (1641, 11),\n",
        " (1642, 22),\n",
        " (1643, 13),\n",
        " (1644, 12),\n",
        " (1645, 13),\n",
        " (1646, 14),\n",
        " (1647, 27),\n",
        " (1648, 33),\n",
        " (1649, 31),\n",
        " (1650, 18),\n",
        " (1651, 11),\n",
        " (1652, 9),\n",
        " (1653, 15),\n",
        " (1654, 8),\n",
        " (1655, 14),\n",
        " (1656, 7),\n",
        " (1657, 9),\n",
        " (1658, 11),\n",
        " (1659, 31),\n",
        " (1660, 20),\n",
        " (1661, 12),\n",
        " (1662, 13),\n",
        " (1663, 16),\n",
        " (1664, 14),\n",
        " (1665, 11),\n",
        " (1666, 8),\n",
        " (1667, 11),\n",
        " (1668, 8),\n",
        " (1669, 8),\n",
        " (1670, 14),\n",
        " (1671, 10),\n",
        " (1672, 10),\n",
        " (1673, 13),\n",
        " (1674, 17),\n",
        " (1675, 21),\n",
        " (1676, 33),\n",
        " (1677, 15),\n",
        " (1678, 17),\n",
        " (1679, 20),\n",
        " (1680, 20),\n",
        " (1681, 16),\n",
        " (1682, 20),\n",
        " (1683, 22),\n",
        " (1684, 14),\n",
        " (1685, 26),\n",
        " (1686, 11),\n",
        " (1687, 19),\n",
        " (1688, 28),\n",
        " (1689, 32),\n",
        " (1690, 28),\n",
        " (1691, 28),\n",
        " (1692, 19),\n",
        " (1693, 28),\n",
        " (1694, 17),\n",
        " (1695, 19),\n",
        " (1696, 16),\n",
        " (1697, 26),\n",
        " (1698, 37),\n",
        " (1699, 25),\n",
        " (1700, 24),\n",
        " (1701, 24),\n",
        " (1702, 25),\n",
        " (1703, 7),\n",
        " (1704, 21),\n",
        " (1705, 16),\n",
        " (1706, 12),\n",
        " (1707, 17),\n",
        " (1708, 15),\n",
        " (1709, 25),\n",
        " (1710, 30),\n",
        " (1711, 22),\n",
        " (1712, 13),\n",
        " (1713, 15),\n",
        " (1714, 15),\n",
        " (1715, 13),\n",
        " (1716, 17),\n",
        " (1717, 21),\n",
        " (1718, 15),\n",
        " (1719, 10),\n",
        " (1720, 36),\n",
        " (1721, 33),\n",
        " (1722, 17),\n",
        " (1723, 17),\n",
        " (1724, 25),\n",
        " (1725, 23),\n",
        " (1726, 30),\n",
        " (1727, 28),\n",
        " (1728, 22),\n",
        " (1729, 21),\n",
        " (1730, 29),\n",
        " (1731, 14),\n",
        " (1732, 24),\n",
        " (1733, 25),\n",
        " (1734, 27),\n",
        " (1735, 13),\n",
        " (1736, 26),\n",
        " (1737, 25),\n",
        " (1738, 30),\n",
        " (1739, 22),\n",
        " (1740, 21),\n",
        " (1741, 29),\n",
        " (1742, 41),\n",
        " (1743, 34),\n",
        " (1744, 37),\n",
        " (1745, 32),\n",
        " (1746, 23),\n",
        " (1747, 32),\n",
        " (1748, 21),\n",
        " (1749, 29),\n",
        " (1750, 34),\n",
        " (1751, 30),\n",
        " (1752, 27),\n",
        " (1753, 26),\n",
        " (1754, 29),\n",
        " (1755, 36),\n",
        " (1756, 38),\n",
        " (1757, 32),\n",
        " (1758, 38),\n",
        " (1759, 34),\n",
        " (1760, 51),\n",
        " (1761, 47),\n",
        " (1762, 33),\n",
        " (1763, 34),\n",
        " (1764, 45),\n",
        " (1765, 35),\n",
        " (1766, 47),\n",
        " (1767, 59),\n",
        " (1768, 42),\n",
        " (1769, 40),\n",
        " (1770, 55),\n",
        " (1771, 59),\n",
        " (1772, 51),\n",
        " (1773, 68),\n",
        " (1774, 72),\n",
        " (1775, 62),\n",
        " (1776, 45),\n",
        " (1777, 66),\n",
        " (1778, 31),\n",
        " (1779, 40),\n",
        " (1780, 52),\n",
        " (1781, 51),\n",
        " (1782, 50),\n",
        " (1783, 45),\n",
        " (1784, 72),\n",
        " (1785, 72),\n",
        " (1786, 83),\n",
        " (1787, 75),\n",
        " (1788, 92),\n",
        " (1789, 77),\n",
        " (1790, 90),\n",
        " (1791, 103),\n",
        " (1792, 115),\n",
        " (1793, 101),\n",
        " (1794, 109),\n",
        " (1795, 118),\n",
        " (1796, 116),\n",
        " (1797, 91),\n",
        " (1798, 127),\n",
        " (1799, 143),\n",
        " (1800, 138),\n",
        " (1801, 116),\n",
        " (1802, 121),\n",
        " (1803, 137),\n",
        " (1804, 148),\n",
        " (1805, 127),\n",
        " (1806, 159),\n",
        " (1807, 141),\n",
        " (1808, 161),\n",
        " (1809, 168),\n",
        " (1810, 189),\n",
        " (1811, 189),\n",
        " (1812, 191),\n",
        " (1813, 186),\n",
        " (1814, 205),\n",
        " (1815, 199),\n",
        " (1816, 193),\n",
        " (1817, 226),\n",
        " (1818, 232),\n",
        " (1819, 208),\n",
        " (1820, 99),\n",
        " (1821, 80),\n",
        " (1822, 90),\n",
        " (1823, 65),\n",
        " (1824, 103),\n",
        " (1825, 87),\n",
        " (1826, 86),\n",
        " (1827, 92),\n",
        " (1828, 84),\n",
        " (1829, 91),\n",
        " (1830, 121),\n",
        " (1831, 110),\n",
        " (1832, 105),\n",
        " (1833, 93),\n",
        " (1834, 106),\n",
        " (1835, 110),\n",
        " (1836, 125),\n",
        " (1837, 123),\n",
        " (1838, 122),\n",
        " (1839, 122),\n",
        " (1840, 123),\n",
        " (1841, 132),\n",
        " (1842, 135),\n",
        " (1843, 104),\n",
        " (1844, 124),\n",
        " (1845, 161),\n",
        " (1846, 146),\n",
        " (1847, 133),\n",
        " (1848, 127),\n",
        " (1849, 151),\n",
        " (1850, 173),\n",
        " (1851, 127),\n",
        " (1852, 171),\n",
        " (1853, 146),\n",
        " (1854, 134),\n",
        " (1855, 134),\n",
        " (1856, 120),\n",
        " (1857, 142),\n",
        " (1858, 132),\n",
        " (1859, 176),\n",
        " (1860, 145),\n",
        " (1861, 91),\n",
        " (1862, 119),\n",
        " (1863, 110),\n",
        " (1864, 153),\n",
        " (1865, 167),\n",
        " (1866, 152),\n",
        " (1867, 166),\n",
        " (1868, 173),\n",
        " (1869, 161),\n",
        " (1870, 132),\n",
        " (1871, 143),\n",
        " (1872, 156),\n",
        " (1873, 159),\n",
        " (1874, 124),\n",
        " (1875, 131),\n",
        " (1876, 172),\n",
        " (1877, 149),\n",
        " (1878, 144),\n",
        " (1879, 157),\n",
        " (1880, 180),\n",
        " (1881, 199),\n",
        " (1882, 194),\n",
        " (1883, 178),\n",
        " (1884, 168),\n",
        " (1885, 174),\n",
        " (1886, 194),\n",
        " (1887, 182),\n",
        " (1888, 182),\n",
        " (1889, 190),\n",
        " (1890, 204),\n",
        " (1891, 195),\n",
        " (1892, 210),\n",
        " (1893, 193),\n",
        " (1894, 199),\n",
        " (1895, 205),\n",
        " (1896, 201),\n",
        " (1897, 211),\n",
        " (1898, 170),\n",
        " (1899, 223),\n",
        " (1900, 231),\n",
        " (1901, 235),\n",
        " (1902, 207),\n",
        " (1903, 228),\n",
        " (1904, 251),\n",
        " (1905, 249),\n",
        " (1906, 235),\n",
        " (1907, 264),\n",
        " (1908, 234),\n",
        " (1909, 305),\n",
        " (1910, 269),\n",
        " (1911, 281),\n",
        " (1912, 303),\n",
        " (1913, 327),\n",
        " (1914, 299),\n",
        " (1915, 323),\n",
        " (1916, 282),\n",
        " (1917, 262),\n",
        " (1918, 222),\n",
        " (1919, 250),\n",
        " (1920, 282),\n",
        " (1921, 259),\n",
        " (1922, 309),\n",
        " (1923, 319),\n",
        " (1924, 314),\n",
        " (1925, 309),\n",
        " (1926, 338),\n",
        " (1927, 322),\n",
        " (1928, 341),\n",
        " (1929, 372),\n",
        " (1930, 377),\n",
        " (1931, 304),\n",
        " (1932, 334),\n",
        " (1933, 265),\n",
        " (1934, 352),\n",
        " (1935, 359),\n",
        " (1936, 401),\n",
        " (1937, 346),\n",
        " (1938, 394),\n",
        " (1939, 389),\n",
        " (1940, 416),\n",
        " (1941, 453),\n",
        " (1942, 344),\n",
        " (1943, 282),\n",
        " (1944, 277),\n",
        " (1945, 277),\n",
        " (1946, 447),\n",
        " (1947, 382),\n",
        " (1948, 371),\n",
        " (1949, 367),\n",
        " (1950, 403),\n",
        " (1951, 361),\n",
        " (1952, 340),\n",
        " (1953, 429),\n",
        " (1954, 394),\n",
        " (1955, 438),\n",
        " (1956, 473),\n",
        " (1957, 423),\n",
        " (1958, 445),\n",
        " (1959, 464),\n",
        " (1960, 510),\n",
        " (1961, 557),\n",
        " (1962, 538),\n",
        " (1963, 553),\n",
        " (1964, 606),\n",
        " (1965, 633),\n",
        " (1966, 670),\n",
        " (1967, 641),\n",
        " (1968, 673),\n",
        " (1969, 766),\n",
        " (1970, 724),\n",
        " (1971, 715),\n",
        " (1972, 858),\n",
        " (1973, 851),\n",
        " (1974, 845),\n",
        " (1975, 1054),\n",
        " (1976, 904),\n",
        " (1977, 833),\n",
        " (1978, 822),\n",
        " (1979, 928),\n",
        " (1980, 860),\n",
        " (1981, 865),\n",
        " (1982, 814),\n",
        " (1983, 947),\n",
        " (1984, 930),\n",
        " (1985, 995),\n",
        " (1986, 981),\n",
        " (1987, 1003),\n",
        " (1988, 1024),\n",
        " (1989, 990),\n",
        " (1990, 967),\n",
        " (1991, 1000),\n",
        " (1992, 1037),\n",
        " (1993, 1402),\n",
        " (1994, 1443),\n",
        " (1995, 1421),\n",
        " (1996, 1530),\n",
        " (1997, 961),\n",
        " (1998, 977),\n",
        " (1999, 922),\n",
        " (2000, 954),\n",
        " (2001, 1002),\n",
        " (2002, 854),\n",
        " (2003, 837),\n",
        " (2004, 824),\n",
        " (2005, 740),\n",
        " (2006, 376),\n",
        " (2007, 542),\n",
        " (2008, 876),\n",
        " (2009, 764),\n",
        " (2010, 804),\n",
        " (2011, 705),\n",
        " (2012, 473),\n",
        " (2013, 32),\n",
        " (2208, 1),\n",
        " (9999, 4)]"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#This is the list we want to convert\n",
      "len(words_df.ix['new']['year_counts'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "435"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Define a function that turns the list of (year, count) tuples into a Pandas.Series with counts as values and years as index"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mk_series = lambda word: Series(map(lambda x: x[1], words_df.ix[word]['year_counts']), index = map(lambda x: x[0], words_df.ix[word]['year_counts'] ))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Check if all entries got preserved\n",
      "len(mk_series('new'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "435"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Seems like the 'None' entries did not get converted. Good!\n",
      "sorted(mk_series('new').index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "[1538.0,\n",
        " 1549.0,\n",
        " 1550.0,\n",
        " 1551.0,\n",
        " 1565.0,\n",
        " 1570.0,\n",
        " 1594.0,\n",
        " 1602.0,\n",
        " 1613.0,\n",
        " 1624.0,\n",
        " 1635.0,\n",
        " 1646.0,\n",
        " 1657.0,\n",
        " 1660.0,\n",
        " 1668.0,\n",
        " 1671.0,\n",
        " 1679.0,\n",
        " 1682.0,\n",
        " 1693.0,\n",
        " 1701.0,\n",
        " 1709.0,\n",
        " 1712.0,\n",
        " 1723.0,\n",
        " 1734.0,\n",
        " 1745.0,\n",
        " 1756.0,\n",
        " 1767.0,\n",
        " 1770.0,\n",
        " 1778.0,\n",
        " 1781.0,\n",
        " 1789.0,\n",
        " 1792.0,\n",
        " 1800.0,\n",
        " 1808.0,\n",
        " 1811.0,\n",
        " 1819.0,\n",
        " 1822.0,\n",
        " 1833.0,\n",
        " 1844.0,\n",
        " 1855.0,\n",
        " 1866.0,\n",
        " 1877.0,\n",
        " 1880.0,\n",
        " 1885.0,\n",
        " 1888.0,\n",
        " 1891.0,\n",
        " 1899.0,\n",
        " 1904.0,\n",
        " 1907.0,\n",
        " 1910.0,\n",
        " 1915.0,\n",
        " 1918.0,\n",
        " 1921.0,\n",
        " 1929.0,\n",
        " 1932.0,\n",
        " 1943.0,\n",
        " 1954.0,\n",
        " 1965.0,\n",
        " 1976.0,\n",
        " nan,\n",
        " 1577.0,\n",
        " 1580.0,\n",
        " 1581.0,\n",
        " 1582.0,\n",
        " 1585.0,\n",
        " 1586.0,\n",
        " 1590.0,\n",
        " 1593.0,\n",
        " 1595.0,\n",
        " 1596.0,\n",
        " 1598.0,\n",
        " 1599.0,\n",
        " 1600.0,\n",
        " 1601.0,\n",
        " 1603.0,\n",
        " 1604.0,\n",
        " 1605.0,\n",
        " 1606.0,\n",
        " 1607.0,\n",
        " 1608.0,\n",
        " 1609.0,\n",
        " 1610.0,\n",
        " 1611.0,\n",
        " 1612.0,\n",
        " 1614.0,\n",
        " 1615.0,\n",
        " 1617.0,\n",
        " 1618.0,\n",
        " 1619.0,\n",
        " 1620.0,\n",
        " 1621.0,\n",
        " 1622.0,\n",
        " 1623.0,\n",
        " 1625.0,\n",
        " 1626.0,\n",
        " 1627.0,\n",
        " 1628.0,\n",
        " 1629.0,\n",
        " 1630.0,\n",
        " 1631.0,\n",
        " 1632.0,\n",
        " 1633.0,\n",
        " 1634.0,\n",
        " 1636.0,\n",
        " 1637.0,\n",
        " 1638.0,\n",
        " 1639.0,\n",
        " 1640.0,\n",
        " 1641.0,\n",
        " 1642.0,\n",
        " 1643.0,\n",
        " 1644.0,\n",
        " 1645.0,\n",
        " 1647.0,\n",
        " 1648.0,\n",
        " 1649.0,\n",
        " 1650.0,\n",
        " 1651.0,\n",
        " 1652.0,\n",
        " 1653.0,\n",
        " 1654.0,\n",
        " 1655.0,\n",
        " 1656.0,\n",
        " 1658.0,\n",
        " 1659.0,\n",
        " 1661.0,\n",
        " 1662.0,\n",
        " 1663.0,\n",
        " 1664.0,\n",
        " 1665.0,\n",
        " 1666.0,\n",
        " 1667.0,\n",
        " 1669.0,\n",
        " 1670.0,\n",
        " 1672.0,\n",
        " 1673.0,\n",
        " 1674.0,\n",
        " 1675.0,\n",
        " 1676.0,\n",
        " 1677.0,\n",
        " 1678.0,\n",
        " 1680.0,\n",
        " 1681.0,\n",
        " 1683.0,\n",
        " 1684.0,\n",
        " 1685.0,\n",
        " 1686.0,\n",
        " 1687.0,\n",
        " 1688.0,\n",
        " 1689.0,\n",
        " 1690.0,\n",
        " 1691.0,\n",
        " 1692.0,\n",
        " 1694.0,\n",
        " 1695.0,\n",
        " 1696.0,\n",
        " 1697.0,\n",
        " 1698.0,\n",
        " 1699.0,\n",
        " 1700.0,\n",
        " 1702.0,\n",
        " 1703.0,\n",
        " 1704.0,\n",
        " 1705.0,\n",
        " 1706.0,\n",
        " 1707.0,\n",
        " 1708.0,\n",
        " 1710.0,\n",
        " 1711.0,\n",
        " 1713.0,\n",
        " 1714.0,\n",
        " 1715.0,\n",
        " 1716.0,\n",
        " 1717.0,\n",
        " 1718.0,\n",
        " 1719.0,\n",
        " 1720.0,\n",
        " 1721.0,\n",
        " 1722.0,\n",
        " 1724.0,\n",
        " 1725.0,\n",
        " 1726.0,\n",
        " 1727.0,\n",
        " 1728.0,\n",
        " 1729.0,\n",
        " 1730.0,\n",
        " 1731.0,\n",
        " 1732.0,\n",
        " 1733.0,\n",
        " 1735.0,\n",
        " 1736.0,\n",
        " 1737.0,\n",
        " 1738.0,\n",
        " 1739.0,\n",
        " 1740.0,\n",
        " 1741.0,\n",
        " 1742.0,\n",
        " 1743.0,\n",
        " 1744.0,\n",
        " 1746.0,\n",
        " 1747.0,\n",
        " 1748.0,\n",
        " 1749.0,\n",
        " 1750.0,\n",
        " 1751.0,\n",
        " 1752.0,\n",
        " 1753.0,\n",
        " 1754.0,\n",
        " 1755.0,\n",
        " 1757.0,\n",
        " 1758.0,\n",
        " 1759.0,\n",
        " 1760.0,\n",
        " 1761.0,\n",
        " 1762.0,\n",
        " 1763.0,\n",
        " 1764.0,\n",
        " 1765.0,\n",
        " 1766.0,\n",
        " 1768.0,\n",
        " 1769.0,\n",
        " 1771.0,\n",
        " 1772.0,\n",
        " 1773.0,\n",
        " 1774.0,\n",
        " 1775.0,\n",
        " 1776.0,\n",
        " 1777.0,\n",
        " 1779.0,\n",
        " 1780.0,\n",
        " 1782.0,\n",
        " 1783.0,\n",
        " 1784.0,\n",
        " 1785.0,\n",
        " 1786.0,\n",
        " 1787.0,\n",
        " 1788.0,\n",
        " 1790.0,\n",
        " 1791.0,\n",
        " 1793.0,\n",
        " 1794.0,\n",
        " 1795.0,\n",
        " 1796.0,\n",
        " 1797.0,\n",
        " 1798.0,\n",
        " 1799.0,\n",
        " 1801.0,\n",
        " 1802.0,\n",
        " 1803.0,\n",
        " 1804.0,\n",
        " 1805.0,\n",
        " 1806.0,\n",
        " 1807.0,\n",
        " 1809.0,\n",
        " 1810.0,\n",
        " 1812.0,\n",
        " 1813.0,\n",
        " 1814.0,\n",
        " 1815.0,\n",
        " 1816.0,\n",
        " 1817.0,\n",
        " 1818.0,\n",
        " 1820.0,\n",
        " 1821.0,\n",
        " 1823.0,\n",
        " 1824.0,\n",
        " 1825.0,\n",
        " 1826.0,\n",
        " 1827.0,\n",
        " 1828.0,\n",
        " 1829.0,\n",
        " 1830.0,\n",
        " 1831.0,\n",
        " 1832.0,\n",
        " 1834.0,\n",
        " 1835.0,\n",
        " 1836.0,\n",
        " 1837.0,\n",
        " 1838.0,\n",
        " 1839.0,\n",
        " 1840.0,\n",
        " 1841.0,\n",
        " 1842.0,\n",
        " 1843.0,\n",
        " 1845.0,\n",
        " 1846.0,\n",
        " 1847.0,\n",
        " 1848.0,\n",
        " 1849.0,\n",
        " 1850.0,\n",
        " 1851.0,\n",
        " 1852.0,\n",
        " 1853.0,\n",
        " 1854.0,\n",
        " 1856.0,\n",
        " 1857.0,\n",
        " 1858.0,\n",
        " 1859.0,\n",
        " 1860.0,\n",
        " 1861.0,\n",
        " 1862.0,\n",
        " 1863.0,\n",
        " 1864.0,\n",
        " 1865.0,\n",
        " 1867.0,\n",
        " 1868.0,\n",
        " 1869.0,\n",
        " 1870.0,\n",
        " 1871.0,\n",
        " 1872.0,\n",
        " 1873.0,\n",
        " 1874.0,\n",
        " 1875.0,\n",
        " 1876.0,\n",
        " 1878.0,\n",
        " 1879.0,\n",
        " 1881.0,\n",
        " 1882.0,\n",
        " 1883.0,\n",
        " 1884.0,\n",
        " 1886.0,\n",
        " 1887.0,\n",
        " 1889.0,\n",
        " 1890.0,\n",
        " 1892.0,\n",
        " 1893.0,\n",
        " 1894.0,\n",
        " 1895.0,\n",
        " 1896.0,\n",
        " 1897.0,\n",
        " 1898.0,\n",
        " 1900.0,\n",
        " 1901.0,\n",
        " 1902.0,\n",
        " 1903.0,\n",
        " 1905.0,\n",
        " 1906.0,\n",
        " 1908.0,\n",
        " 1909.0,\n",
        " 1911.0,\n",
        " 1912.0,\n",
        " 1913.0,\n",
        " 1914.0,\n",
        " 1916.0,\n",
        " 1917.0,\n",
        " 1919.0,\n",
        " 1920.0,\n",
        " 1922.0,\n",
        " 1923.0,\n",
        " 1924.0,\n",
        " 1925.0,\n",
        " 1926.0,\n",
        " 1927.0,\n",
        " 1928.0,\n",
        " 1930.0,\n",
        " 1931.0,\n",
        " 1933.0,\n",
        " 1934.0,\n",
        " 1935.0,\n",
        " 1936.0,\n",
        " 1937.0,\n",
        " 1938.0,\n",
        " 1939.0,\n",
        " 1940.0,\n",
        " 1941.0,\n",
        " 1942.0,\n",
        " 1944.0,\n",
        " 1945.0,\n",
        " 1946.0,\n",
        " 1947.0,\n",
        " 1948.0,\n",
        " 1949.0,\n",
        " 1950.0,\n",
        " 1951.0,\n",
        " 1952.0,\n",
        " 1953.0,\n",
        " 1955.0,\n",
        " 1956.0,\n",
        " 1957.0,\n",
        " 1958.0,\n",
        " 1959.0,\n",
        " 1960.0,\n",
        " 1961.0,\n",
        " 1962.0,\n",
        " 1963.0,\n",
        " 1964.0,\n",
        " 1966.0,\n",
        " 1967.0,\n",
        " 1968.0,\n",
        " 1969.0,\n",
        " 1970.0,\n",
        " 1971.0,\n",
        " 1972.0,\n",
        " 1973.0,\n",
        " 1974.0,\n",
        " 1975.0,\n",
        " 1977.0,\n",
        " 1978.0,\n",
        " 1979.0,\n",
        " 1980.0,\n",
        " 1981.0,\n",
        " 1982.0,\n",
        " 1983.0,\n",
        " 1984.0,\n",
        " 1985.0,\n",
        " 1986.0,\n",
        " 1987.0,\n",
        " 1988.0,\n",
        " 1989.0,\n",
        " 1990.0,\n",
        " 1991.0,\n",
        " 1992.0,\n",
        " 1993.0,\n",
        " 1994.0,\n",
        " 1995.0,\n",
        " 1996.0,\n",
        " 1997.0,\n",
        " 1998.0,\n",
        " 1999.0,\n",
        " 2000.0,\n",
        " 2001.0,\n",
        " 2002.0,\n",
        " 2003.0,\n",
        " 2004.0,\n",
        " 2005.0,\n",
        " 2006.0,\n",
        " 2007.0,\n",
        " 2008.0,\n",
        " 2009.0,\n",
        " 2010.0,\n",
        " 2011.0,\n",
        " 2012.0,\n",
        " 2013.0,\n",
        " 2208.0,\n",
        " 9999.0]"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def mk_clean_series(year_counts):\n",
      "    if year_counts:\n",
      "        s['count'] = 0\n",
      "        #s = DataFrame.from_records(year_counts, columns =['year', 'count'])\n",
      "        #s = s.set_index('year')\n",
      "        #s = s[s['count'] <= MAX_YEAR]\n",
      "        return s['count']\n",
      "    else:\n",
      "        return None"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "type(mk_clean_series(words_df.ix['new']['year_counts']))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "pandas.core.series.Series"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#I dont understand what the items with index 'None' or != 'None' mean, ignore them for now\n",
      "a = mk_series('new')\n",
      "b = a[a.index <= MAX_YEAR]\n",
      "c = a[a.index > MAX_YEAR]\n",
      "d = a[a.index == None]\n",
      "e = a[a.index != None]\n",
      "print len(b), len(c), len(a)\n",
      "print d, e"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "432 2 435\n",
        "90 804\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#words_df['occ_series'] = words_df['year_counts'].apply(mk_clean_series)\n",
      "frequent = words_df[words_df['total'] > 25000].copy()\n",
      "len(frequent)\n",
      "frequent.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>average</th>\n",
        "      <th>num_years</th>\n",
        "      <th>total</th>\n",
        "      <th>word</th>\n",
        "      <th>year_counts</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>key</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>art</th>\n",
        "      <td>  66.661538</td>\n",
        "      <td> 390</td>\n",
        "      <td> 25998</td>\n",
        "      <td>         art</td>\n",
        "      <td> [(1867, 24), (1573, 1), (1584, 2), (1650, 3), ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>book</th>\n",
        "      <td>  88.596535</td>\n",
        "      <td> 404</td>\n",
        "      <td> 35793</td>\n",
        "      <td>        book</td>\n",
        "      <td> [(1611, 1), (2010, 291), (2007, 253), (1996, 4...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>development</th>\n",
        "      <td> 202.994709</td>\n",
        "      <td> 189</td>\n",
        "      <td> 38366</td>\n",
        "      <td> development</td>\n",
        "      <td> [(1866, 1), (1855, 7), (1819, 2), (, 23), (201...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>public</th>\n",
        "      <td>  95.604167</td>\n",
        "      <td> 288</td>\n",
        "      <td> 27534</td>\n",
        "      <td>      public</td>\n",
        "      <td> [(1774, 14), (2005, 269), (1994, 479), (1983, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>act</th>\n",
        "      <td>  87.139344</td>\n",
        "      <td> 366</td>\n",
        "      <td> 31893</td>\n",
        "      <td>         act</td>\n",
        "      <td> [(1725, 6), (1714, 9), (1703, 4), (1695, 10), ...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "                average  num_years  total         word  \\\n",
        "key                                                      \n",
        "art           66.661538        390  25998          art   \n",
        "book          88.596535        404  35793         book   \n",
        "development  202.994709        189  38366  development   \n",
        "public        95.604167        288  27534       public   \n",
        "act           87.139344        366  31893          act   \n",
        "\n",
        "                                                   year_counts  \n",
        "key                                                             \n",
        "art          [(1867, 24), (1573, 1), (1584, 2), (1650, 3), ...  \n",
        "book         [(1611, 1), (2010, 291), (2007, 253), (1996, 4...  \n",
        "development  [(1866, 1), (1855, 7), (1819, 2), (, 23), (201...  \n",
        "public       [(1774, 14), (2005, 269), (1994, 479), (1983, ...  \n",
        "act          [(1725, 6), (1714, 9), (1703, 4), (1695, 10), ...  "
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#frequent['occ_series'] = frequent['year_counts'].apply(tuple)\n",
      "frequent['occ_series'] = frequent['year_counts'].apply(mk_clean_series)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "AssertionError",
       "evalue": "Wrong number of items passed (1 vs 435)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-37-9d95ad56f62d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#frequent['occ_series'] = frequent['year_counts'].apply(tuple)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mfrequent\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'occ_series'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfrequent\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'year_counts'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmk_clean_series\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__setitem__\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   2072\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2073\u001b[0m             \u001b[1;31m# set column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2074\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2075\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2076\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_boolean_set\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_set_item\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   2108\u001b[0m         \"\"\"\n\u001b[0;32m   2109\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2110\u001b[1;33m         \u001b[0mNDFrame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2111\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2112\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0minsert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/generic.pyc\u001b[0m in \u001b[0;36m_set_item\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m    544\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    545\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 546\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    547\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    548\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mset\u001b[1;34m(self, item, value)\u001b[0m\n\u001b[0;32m    969\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    970\u001b[0m             \u001b[1;31m# insert at end\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 971\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    972\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    973\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_known_consolidated\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/internals.pyc\u001b[0m in \u001b[0;36minsert\u001b[1;34m(self, loc, item, value)\u001b[0m\n\u001b[0;32m    981\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    982\u001b[0m         \u001b[1;31m# new block\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 983\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_add_new_block\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    984\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    985\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m_add_new_block\u001b[1;34m(self, item, value, loc)\u001b[0m\n\u001b[0;32m   1010\u001b[0m             \u001b[0mloc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1011\u001b[0m         new_block = make_block(value, self.items[loc:loc + 1].copy(),\n\u001b[1;32m-> 1012\u001b[1;33m                                self.items)\n\u001b[0m\u001b[0;32m   1013\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_block\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1014\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mmake_block\u001b[1;34m(values, items, ref_items)\u001b[0m\n\u001b[0;32m    481\u001b[0m         \u001b[0mklass\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mObjectBlock\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    482\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 483\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mklass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitems\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mref_items\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    484\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    485\u001b[0m \u001b[1;31m# TODO: flexible with index=None and/or items=None\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas-0.10.1-py2.7-linux-i686.egg/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, values, items, ref_items, ndim)\u001b[0m\n\u001b[0;32m     31\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m             raise AssertionError('Wrong number of items passed (%d vs %d)'\n\u001b[1;32m---> 33\u001b[1;33m                                  % (len(items), len(values)))\n\u001b[0m\u001b[0;32m     34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ref_locs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mAssertionError\u001b[0m: Wrong number of items passed (1 vs 435)"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "frequent.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>average</th>\n",
        "      <th>num_years</th>\n",
        "      <th>total</th>\n",
        "      <th>word</th>\n",
        "      <th>year_counts</th>\n",
        "      <th>occ_series</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>key</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>art</th>\n",
        "      <td>  66.661538</td>\n",
        "      <td> 390</td>\n",
        "      <td> 25998</td>\n",
        "      <td>         art</td>\n",
        "      <td> [(1867, 24), (1573, 1), (1584, 2), (1650, 3), ...</td>\n",
        "      <td> ((1867, 24), (1573, 1), (1584, 2), (1650, 3), ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>book</th>\n",
        "      <td>  88.596535</td>\n",
        "      <td> 404</td>\n",
        "      <td> 35793</td>\n",
        "      <td>        book</td>\n",
        "      <td> [(1611, 1), (2010, 291), (2007, 253), (1996, 4...</td>\n",
        "      <td> ((1611, 1), (2010, 291), (2007, 253), (1996, 4...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>development</th>\n",
        "      <td> 202.994709</td>\n",
        "      <td> 189</td>\n",
        "      <td> 38366</td>\n",
        "      <td> development</td>\n",
        "      <td> [(1866, 1), (1855, 7), (1819, 2), (, 23), (201...</td>\n",
        "      <td> ((1866, 1), (1855, 7), (1819, 2), (, 23), (201...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>public</th>\n",
        "      <td>  95.604167</td>\n",
        "      <td> 288</td>\n",
        "      <td> 27534</td>\n",
        "      <td>      public</td>\n",
        "      <td> [(1774, 14), (2005, 269), (1994, 479), (1983, ...</td>\n",
        "      <td> ((1774, 14), (2005, 269), (1994, 479), (1983, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>act</th>\n",
        "      <td>  87.139344</td>\n",
        "      <td> 366</td>\n",
        "      <td> 31893</td>\n",
        "      <td>         act</td>\n",
        "      <td> [(1725, 6), (1714, 9), (1703, 4), (1695, 10), ...</td>\n",
        "      <td> ((1725, 6), (1714, 9), (1703, 4), (1695, 10), ...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "output_type": "pyout",
       "prompt_number": 34,
       "text": [
        "                average  num_years  total         word  \\\n",
        "key                                                      \n",
        "art           66.661538        390  25998          art   \n",
        "book          88.596535        404  35793         book   \n",
        "development  202.994709        189  38366  development   \n",
        "public        95.604167        288  27534       public   \n",
        "act           87.139344        366  31893          act   \n",
        "\n",
        "                                                   year_counts  \\\n",
        "key                                                              \n",
        "art          [(1867, 24), (1573, 1), (1584, 2), (1650, 3), ...   \n",
        "book         [(1611, 1), (2010, 291), (2007, 253), (1996, 4...   \n",
        "development  [(1866, 1), (1855, 7), (1819, 2), (, 23), (201...   \n",
        "public       [(1774, 14), (2005, 269), (1994, 479), (1983, ...   \n",
        "act          [(1725, 6), (1714, 9), (1703, 4), (1695, 10), ...   \n",
        "\n",
        "                                                    occ_series  \n",
        "key                                                             \n",
        "art          ((1867, 24), (1573, 1), (1584, 2), (1650, 3), ...  \n",
        "book         ((1611, 1), (2010, 291), (2007, 253), (1996, 4...  \n",
        "development  ((1866, 1), (1855, 7), (1819, 2), (, 23), (201...  \n",
        "public       ((1774, 14), (2005, 269), (1994, 479), (1983, ...  \n",
        "act          ((1725, 6), (1714, 9), (1703, 4), (1695, 10), ...  "
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import matplotlib.pyplot as plt\n",
      "words_df['total'][words_df['total'] > 100].value_counts().hist(bins=20)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words_df.columns[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Adding Standard Deviation to Table"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def std_dev(row):\n",
      "    n = 0\n",
      "    running_sum = 0\n",
      "    for tt in row['year_counts']:\n",
      "        n += 1\n",
      "        running_sum += (tt[1] - row['average'])**2\n",
      "    return (running_sum/n)**(0.5)\n",
      "\n",
      "def skewness\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words_df['std_dev'] = words_df.apply(std_dev, axis = 1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Words with the highest Standard Deviation"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words_df[words_df['total'] > 100].sort(columns = 'std_dev', ascending = False).head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Words with the highest total count"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words_df.sort(columns = 'total', ascending = False).head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Words with the highest average count"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Words with the highest maximum"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sorted(words_stats, key = lambda x: x[3], reverse=True)[:10]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sorted(words_stats, key = lambda x: x[4], reverse=True)[:20]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}