NLP_biden_inauguration_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_table(\"biden_speech.rtf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>{\\rtf1\\ansi\\ansicpg1252\\cocoartf2577</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\\cocoatextscaling0\\cocoaplatform0{\\fonttbl\\f0\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{\\colortbl;\\red255\\green255\\blue255;\\red0\\gree...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{\\*\\expandedcolortbl;;\\cssrgb\\c0\\c0\\c0;}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\paperw11900\\paperh16840\\margl1440\\margr1440\\v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\deftab720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>Sustained by faith.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>Driven by conviction.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>And, devoted to one another and to this countr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>May God bless America and may God protect our ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>Thank you, America.}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>215 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  {\\rtf1\\ansi\\ansicpg1252\\cocoartf2577\n",
       "0    \\cocoatextscaling0\\cocoaplatform0{\\fonttbl\\f0\\...\n",
       "1    {\\colortbl;\\red255\\green255\\blue255;\\red0\\gree...\n",
       "2             {\\*\\expandedcolortbl;;\\cssrgb\\c0\\c0\\c0;}\n",
       "3    \\paperw11900\\paperh16840\\margl1440\\margr1440\\v...\n",
       "4                                           \\deftab720\n",
       "..                                                 ...\n",
       "210                               Sustained by faith.\\\n",
       "211                             Driven by conviction.\\\n",
       "212  And, devoted to one another and to this countr...\n",
       "213  May God bless America and may God protect our ...\n",
       "214                               Thank you, America.}\n",
       "\n",
       "[215 rows x 1 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df.drop(df.index[0:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.columns = ['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>This is America\\'92s day.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>This is democracy\\'92s day.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>A day of history and hope.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Of renewal and resolve.\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Through a crucible for the ages America has be...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text\n",
       "8                          This is America\\'92s day.\\\n",
       "9                        This is democracy\\'92s day.\\\n",
       "10                        A day of history and hope.\\\n",
       "11                           Of renewal and resolve.\\\n",
       "12  Through a crucible for the ages America has be..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['text'] = df1['text'].str.replace(\"\\\\\",\" \")\n",
    "df1['text'] = df1['text'].str.replace(\"}\",\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>expressions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>This is America '92s day.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>This is democracy '92s day.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>A day of history and hope.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Of renewal and resolve.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Through a crucible for the ages America has be...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>Sustained by faith.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>Driven by conviction.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>And, devoted to one another and to this countr...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>May God bless America and may God protect our ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>Thank you, America.</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>207 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text expressions\n",
       "8                           This is America '92s day.         None\n",
       "9                         This is democracy '92s day.         None\n",
       "10                         A day of history and hope.         None\n",
       "11                            Of renewal and resolve.         None\n",
       "12   Through a crucible for the ages America has be...        None\n",
       "..                                                 ...         ...\n",
       "210                               Sustained by faith.         None\n",
       "211                             Driven by conviction.         None\n",
       "212  And, devoted to one another and to this countr...        None\n",
       "213  May God bless America and may God protect our ...        None\n",
       "214                                Thank you, America.        None\n",
       "\n",
       "[207 rows x 2 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#conda install -c conda-forge WordCloud"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main words in the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['polarity'] = df1['text'].apply(polarity_calc)\n",
    "df1['subjectivity'] = df1['text'].apply(subjectivity_calc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "Biden_sent = df1.sort_values(by='polarity', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>polarity</th>\n",
       "      <th>subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>'93The work and prayers uc0 u8232 of centurie...</td>\n",
       "      <td>(1.0, 0.3)</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>If we do this then when our days are through o...</td>\n",
       "      <td>(1.0, 0.3)</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>I promise you this: as the Bible says weeping ...</td>\n",
       "      <td>(0.8, 0.2)</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Through the Civil War, the Great Depression, W...</td>\n",
       "      <td>(0.8, 0.75)</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Here we stand looking out to the great Mall wh...</td>\n",
       "      <td>(0.8, 0.75)</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Uniting to fight the common foes we face:</td>\n",
       "      <td>(-0.3, 0.5)</td>\n",
       "      <td>-0.30</td>\n",
       "      <td>0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>Recent weeks and months have taught us a painf...</td>\n",
       "      <td>(-0.35, 0.575)</td>\n",
       "      <td>-0.35</td>\n",
       "      <td>0.575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>They healed a broken land. uc0 u8232 My fellow...</td>\n",
       "      <td>(-0.4, 0.4)</td>\n",
       "      <td>-0.40</td>\n",
       "      <td>0.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>We have never, ever, ever failed in America wh...</td>\n",
       "      <td>(-0.5, 0.3)</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>0.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Anger, resentment, hatred.</td>\n",
       "      <td>(-0.7, 0.2)</td>\n",
       "      <td>-0.70</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>207 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text       sentiment  \\\n",
       "184   '93The work and prayers uc0 u8232 of centurie...      (1.0, 0.3)   \n",
       "186  If we do this then when our days are through o...      (1.0, 0.3)   \n",
       "149  I promise you this: as the Bible says weeping ...      (0.8, 0.2)   \n",
       "73   Through the Civil War, the Great Depression, W...     (0.8, 0.75)   \n",
       "101  Here we stand looking out to the great Mall wh...     (0.8, 0.75)   \n",
       "..                                                 ...             ...   \n",
       "55          Uniting to fight the common foes we face:      (-0.3, 0.5)   \n",
       "129  Recent weeks and months have taught us a painf...  (-0.35, 0.575)   \n",
       "188  They healed a broken land. uc0 u8232 My fellow...     (-0.4, 0.4)   \n",
       "86   We have never, ever, ever failed in America wh...     (-0.5, 0.3)   \n",
       "56                         Anger, resentment, hatred.      (-0.7, 0.2)   \n",
       "\n",
       "     polarity  subjectivity  \n",
       "184      1.00         0.300  \n",
       "186      1.00         0.300  \n",
       "149      0.80         0.200  \n",
       "73       0.80         0.750  \n",
       "101      0.80         0.750  \n",
       "..        ...           ...  \n",
       "55      -0.30         0.500  \n",
       "129     -0.35         0.575  \n",
       "188     -0.40         0.400  \n",
       "86      -0.50         0.300  \n",
       "56      -0.70         0.200  \n",
       "\n",
       "[207 rows x 4 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Biden_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud, STOPWORDS\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cloudword(text):\n",
    "    try:\n",
    "        text = \"\"\n",
    "        for user,content in text.items():\n",
    "            text += content\n",
    "        fig,ax = plt.subplots(1,1,figsize=(8,8))\n",
    "        wordcloud =  WordCloud(stopwords=STOPWORDS,background_color='white',width=3000,height=3000).generate(text)\n",
    "        plt.imshow(wordcloud)          \n",
    "        plt.axis('off')\n",
    "        plt.show()\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cloudword(text):\n",
    "    try:\n",
    "        return WordCloud(stopwords=STOPWORDS,background_color='white',width=3000,height=3000).generate(text)\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Image data of dtype object cannot be converted to float",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-51bfe41c5d3a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcloudword\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'off'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, filternorm, filterrad, resample, url, data, **kwargs)\u001b[0m\n\u001b[1;32m   2722\u001b[0m         \u001b[0mfilternorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilterrad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2723\u001b[0m         data=None, **kwargs):\n\u001b[0;32m-> 2724\u001b[0;31m     __ret = gca().imshow(\n\u001b[0m\u001b[1;32m   2725\u001b[0m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcmap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maspect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maspect\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2726\u001b[0m         \u001b[0minterpolation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minterpolation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1436\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1437\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1438\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msanitize_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1439\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1440\u001b[0m         \u001b[0mbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_sig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(self, X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, filternorm, filterrad, resample, url, **kwargs)\u001b[0m\n\u001b[1;32m   5521\u001b[0m                               resample=resample, **kwargs)\n\u001b[1;32m   5522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5523\u001b[0;31m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5524\u001b[0m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_alpha\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5525\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_clip_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/matplotlib/image.py\u001b[0m in \u001b[0;36mset_data\u001b[0;34m(self, A)\u001b[0m\n\u001b[1;32m    698\u001b[0m         if (self._A.dtype != np.uint8 and\n\u001b[1;32m    699\u001b[0m                 not np.can_cast(self._A.dtype, float, \"same_kind\")):\n\u001b[0;32m--> 700\u001b[0;31m             raise TypeError(\"Image data of dtype {} cannot be converted to \"\n\u001b[0m\u001b[1;32m    701\u001b[0m                             \"float\".format(self._A.dtype))\n\u001b[1;32m    702\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Image data of dtype object cannot be converted to float"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAD8CAYAAACVSwr3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMX0lEQVR4nO3bX4il9X3H8fenuxEak0aJk5DuKt2WNbotWnRiJPSPaWizay6WgBdqqFQCixBDLpVCk4I3zUUhBP8siyySm+xNJN0UEyktiQVr4yz4bxVlulKdrOAaQwoGKqvfXsxpc3q+szvPrGfO2cH3CwbmeZ7fOefLMOc9zzzzTKoKSRr3G/MeQNL5xzBIagyDpMYwSGoMg6TGMEhq1g1DksNJXk/y3BmOJ8m3kywneSbJNdMfU9IsDTljeAjYe5bj+4Ddo48DwAPvfSxJ87RuGKrqMeDNsyzZD3ynVj0BXJTkE9MaUNLsbZ/Cc+wAXh3bXhnte21yYZIDrJ5VcOGFF157xRVXTOHlJZ3JsWPH3qiqhY0+bhphyBr71rzPuqoOAYcAFhcXa2lpaQovL+lMkvznuTxuGn+VWAEuHdveCZycwvNKmpNphOEocNvorxPXA7+sqvZrhKStY91fJZJ8F7gBuCTJCvAN4AMAVXUQeAS4EVgGfgXcvlnDSpqNdcNQVbesc7yAr0xtIklz552PkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySmkFhSLI3yYtJlpPcvcbxjyT5QZKnkxxPcvv0R5U0K+uGIck24D5gH7AHuCXJnollXwGer6qrgRuAv09ywZRnlTQjQ84YrgOWq+pEVb0NHAH2T6wp4MNJAnwIeBM4PdVJJc3MkDDsAF4d214Z7Rt3L3AlcBJ4FvhaVb07+URJDiRZSrJ06tSpcxxZ0mYbEoassa8mtj8PPAX8NvCHwL1Jfqs9qOpQVS1W1eLCwsIGR5U0K0PCsAJcOra9k9Uzg3G3Aw/XqmXgZeCK6YwoadaGhOFJYHeSXaMLijcDRyfWvAJ8DiDJx4FPAiemOaik2dm+3oKqOp3kTuBRYBtwuKqOJ7ljdPwgcA/wUJJnWf3V466qemMT55a0idYNA0BVPQI8MrHv4NjnJ4G/mO5okubFOx8lNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVIzKAxJ9iZ5MclykrvPsOaGJE8lOZ7kJ9MdU9IsbV9vQZJtwH3AnwMrwJNJjlbV82NrLgLuB/ZW1StJPrZJ80qagSFnDNcBy1V1oqreBo4A+yfW3Ao8XFWvAFTV69MdU9IsDQnDDuDVse2V0b5xlwMXJ/lxkmNJblvriZIcSLKUZOnUqVPnNrGkTTckDFljX01sbweuBb4AfB74mySXtwdVHaqqxapaXFhY2PCwkmZj3WsMrJ4hXDq2vRM4ucaaN6rqLeCtJI8BVwMvTWVKSTM15IzhSWB3kl1JLgBuBo5OrPkH4I+TbE/yQeDTwAvTHVXSrKx7xlBVp5PcCTwKbAMOV9XxJHeMjh+sqheS/Ah4BngXeLCqntvMwSVtnlRNXi6YjcXFxVpaWprLa0vvF0mOVdXiRh/nnY+SGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJKaQWFIsjfJi0mWk9x9lnWfSvJOkpumN6KkWVs3DEm2AfcB+4A9wC1J9pxh3TeBR6c9pKTZGnLGcB2wXFUnqupt4Aiwf411XwW+B7w+xfkkzcGQMOwAXh3bXhnt+z9JdgBfBA6e7YmSHEiylGTp1KlTG51V0owMCUPW2FcT298C7qqqd872RFV1qKoWq2pxYWFh4IiSZm37gDUrwKVj2zuBkxNrFoEjSQAuAW5Mcrqqvj+NISXN1pAwPAnsTrIL+BlwM3Dr+IKq2vW/nyd5CPhHoyBtXeuGoapOJ7mT1b82bAMOV9XxJHeMjp/1uoKkrWfIGQNV9QjwyMS+NYNQVX/13seSNE/e+SipMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkppBYUiyN8mLSZaT3L3G8S8leWb08XiSq6c/qqRZWTcMSbYB9wH7gD3ALUn2TCx7GfjTqroKuAc4NO1BJc3OkDOG64DlqjpRVW8DR4D94wuq6vGq+sVo8wlg53THlDRLQ8KwA3h1bHtltO9Mvgz8cK0DSQ4kWUqydOrUqeFTSpqpIWHIGvtqzYXJZ1kNw11rHa+qQ1W1WFWLCwsLw6eUNFPbB6xZAS4d294JnJxclOQq4EFgX1X9fDrjSZqHIWcMTwK7k+xKcgFwM3B0fEGSy4CHgb+sqpemP6akWVr3jKGqTie5E3gU2AYcrqrjSe4YHT8IfB34KHB/EoDTVbW4eWNL2kypWvNywaZbXFyspaWluby29H6R5Ni5/JD2zkdJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBknNoDAk2ZvkxSTLSe5e43iSfHt0/Jkk10x/VEmzsm4YkmwD7gP2AXuAW5LsmVi2D9g9+jgAPDDlOSXN0JAzhuuA5ao6UVVvA0eA/RNr9gPfqVVPABcl+cSUZ5U0I9sHrNkBvDq2vQJ8esCaHcBr44uSHGD1jALgv5M8t6Fp5+sS4I15DzHQVpoVtta8W2lWgE+ey4OGhCFr7KtzWENVHQIOASRZqqrFAa9/XthK826lWWFrzbuVZoXVec/lcUN+lVgBLh3b3gmcPIc1kraIIWF4EtidZFeSC4CbgaMTa44Ct43+OnE98Muqem3yiSRtDev+KlFVp5PcCTwKbAMOV9XxJHeMjh8EHgFuBJaBXwG3D3jtQ+c89XxspXm30qywtebdSrPCOc6bqnYpQNL7nHc+SmoMg6Rm08OwlW6nHjDrl0YzPpPk8SRXz2POsXnOOu/Yuk8leSfJTbOcb2KGdWdNckOSp5IcT/KTWc84Mct63wsfSfKDJE+P5h1yXW1TJDmc5PUz3Rd0Tu+xqtq0D1YvVv4H8LvABcDTwJ6JNTcCP2T1XojrgX/fzJne46yfAS4efb5vXrMOnXds3b+weoH4pvN1VuAi4HngstH2x87nry3w18A3R58vAG8CF8xp3j8BrgGeO8PxDb/HNvuMYSvdTr3urFX1eFX9YrT5BKv3a8zLkK8twFeB7wGvz3K4CUNmvRV4uKpeAaiq833eAj6cJMCHWA3D6dmOORqk6rHR65/Jht9jmx2GM90qvdE1s7DROb7MaoXnZd15k+wAvggcnOFcaxnytb0cuDjJj5McS3LbzKbrhsx7L3AlqzfyPQt8rarenc14G7bh99iQW6Lfi6ndTj0Dg+dI8llWw/BHmzrR2Q2Z91vAXVX1zuoPtrkZMut24Frgc8BvAv+W5Imqemmzh1vDkHk/DzwF/Bnwe8A/JfnXqvqvTZ7tXGz4PbbZYdhKt1MPmiPJVcCDwL6q+vmMZlvLkHkXgSOjKFwC3JjkdFV9fyYT/trQ74M3quot4K0kjwFXA/MIw5B5bwf+rlZ/iV9O8jJwBfDT2Yy4IRt/j23yRZHtwAlgF7++iPP7E2u+wP+/MPLTOV3AGTLrZaze3fmZecy40Xkn1j/E/C4+DvnaXgn882jtB4HngD84j+d9APjb0ecfB34GXDLH74ff4cwXHzf8HtvUM4bavNup5zXr14GPAvePfgqfrjn9p93Aec8LQ2atqheS/Ah4BngXeLCq5vJv+QO/tvcADyV5ltU33F1VNZd/x07yXeAG4JIkK8A3gA+Mzbrh95i3REtqvPNRUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUvM/YA1djYGMYyEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(cloudword)          \n",
    "plt.axis('off')           \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "  \n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Processing Text tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expression(text):\n",
    "    try:\n",
    "        return TextBlob(text).noun_phrases\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "They healed a broken land. uc0 u8232 My fellow Americans, I close today where I began, with a sacred oath. \n"
     ]
    }
   ],
   "source": [
    "df188=df1.text[188]\n",
    "print(df188)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList(['They', 'healed', 'a', 'broken', 'land', 'uc0', 'u8232', 'My', 'fellow', 'Americans', 'I', 'close', 'today', 'where', 'I', 'began', 'with', 'a', 'sacred', 'oath'])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob188=TextBlob(df188)\n",
    "blob188.words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Error loading brown: <urlopen error [Errno 8] nodename nor\n",
      "[nltk_data]     servname provided, or not known>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('brown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "**********************************************************************\n",
      "  Resource \u001b[93mbrown\u001b[0m not found.\n",
      "  Please use the NLTK Downloader to obtain the resource:\n",
      "\n",
      "  \u001b[31m>>> import nltk\n",
      "  >>> nltk.download('brown')\n",
      "  \u001b[0m\n",
      "  For more information see: https://www.nltk.org/data.html\n",
      "\n",
      "  Attempted to load \u001b[93mcorpora/brown\u001b[0m\n",
      "\n",
      "  Searched in:\n",
      "    - '/Users/pablofernandezrodriguez/nltk_data'\n",
      "    - '/opt/anaconda3/nltk_data'\n",
      "    - '/opt/anaconda3/share/nltk_data'\n",
      "    - '/opt/anaconda3/lib/nltk_data'\n",
      "    - '/usr/share/nltk_data'\n",
      "    - '/usr/local/share/nltk_data'\n",
      "    - '/usr/lib/nltk_data'\n",
      "    - '/usr/local/lib/nltk_data'\n",
      "**********************************************************************\n",
      "\n"
     ]
    },
    {
     "ename": "MissingCorpusError",
     "evalue": "\nLooks like you are missing some required data for this feature.\n\nTo download the necessary data, simply run\n\n    python -m textblob.download_corpora\n\nor use the NLTK downloader to download the missing data: http://nltk.org/data.html\nIf this doesn't fix the problem, file an issue at https://github.com/sloria/TextBlob/issues.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     82\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                     \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{}/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/data.py\u001b[0m in \u001b[0;36mfind\u001b[0;34m(resource_name, paths)\u001b[0m\n\u001b[1;32m    584\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\\n%s\\n%s\\n%s\\n\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 585\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    586\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mbrown\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('brown')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mcorpora/brown.zip/brown/\u001b[0m\n\n  Searched in:\n    - '/Users/pablofernandezrodriguez/nltk_data'\n    - '/opt/anaconda3/nltk_data'\n    - '/opt/anaconda3/share/nltk_data'\n    - '/opt/anaconda3/lib/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n**********************************************************************\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/decorators.py\u001b[0m in \u001b[0;36mdecorated\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/en/np_extractors.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    108\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m         \u001b[0mtrain_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbrown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtagged_sents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'news'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    110\u001b[0m         regexp_tagger = nltk.RegexpTagger([\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, attr)\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m         \u001b[0;31m# This looks circular, but its not, since __load() changes our\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     84\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m                 \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{}/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/nltk/data.py\u001b[0m in \u001b[0;36mfind\u001b[0;34m(resource_name, paths)\u001b[0m\n\u001b[1;32m    584\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\\n%s\\n%s\\n%s\\n\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 585\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    586\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mbrown\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('brown')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mcorpora/brown\u001b[0m\n\n  Searched in:\n    - '/Users/pablofernandezrodriguez/nltk_data'\n    - '/opt/anaconda3/nltk_data'\n    - '/opt/anaconda3/share/nltk_data'\n    - '/opt/anaconda3/lib/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n**********************************************************************\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mMissingCorpusError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-aedef1fb6c54>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mblob188\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnoun_phrases\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/decorators.py\u001b[0m in \u001b[0;36m__get__\u001b[0;34m(self, obj, cls)\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/blob.py\u001b[0m in \u001b[0;36mnoun_phrases\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    466\u001b[0m         \u001b[0;34m\"\"\"Returns a list of noun phrases for this blob.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m         return WordList([phrase.strip().lower()\n\u001b[0;32m--> 468\u001b[0;31m                         \u001b[0;32mfor\u001b[0m \u001b[0mphrase\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnp_extractor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m                         if len(phrase) > 1])\n\u001b[1;32m    470\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/en/np_extractors.py\u001b[0m in \u001b[0;36mextract\u001b[0;34m(self, sentence)\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;34m'''Return a list of noun phrases (strings) for body of text.'''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trained\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tokenize_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m         \u001b[0mtagged\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/textblob/decorators.py\u001b[0m in \u001b[0;36mdecorated\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mMissingCorpusError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdecorated\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMissingCorpusError\u001b[0m: \nLooks like you are missing some required data for this feature.\n\nTo download the necessary data, simply run\n\n    python -m textblob.download_corpora\n\nor use the NLTK downloader to download the missing data: http://nltk.org/data.html\nIf this doesn't fix the problem, file an issue at https://github.com/sloria/TextBlob/issues.\n"
     ]
    }
   ],
   "source": [
    "blob188.noun_phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'they': 1,\n",
       "             'healed': 1,\n",
       "             'a': 2,\n",
       "             'broken': 1,\n",
       "             'land': 1,\n",
       "             'uc0': 1,\n",
       "             'u8232': 1,\n",
       "             'my': 1,\n",
       "             'fellow': 1,\n",
       "             'americans': 1,\n",
       "             'i': 2,\n",
       "             'close': 1,\n",
       "             'today': 1,\n",
       "             'where': 1,\n",
       "             'began': 1,\n",
       "             'with': 1,\n",
       "             'sacred': 1,\n",
       "             'oath': 1})"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob188.word_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attempt to import the whole text together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "biden_speech.rtf\n"
     ]
    }
   ],
   "source": [
    "fulltext = input(\"biden_speech.rtf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fulltext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Back to the analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TextBlob(\"They healed a broken land. up u8232 By fellow Americans, I close today where I began, with a sacred oath. \")"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob188.correct()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract contiguous sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#conda install -c conda-forge spacy textacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from textacy.extract import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = nlp(df188)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_grams=2\n",
    "min_freq=1\n",
    "pd.Series([n.text for n in ngram(text, n=n_grams, min_freq=1)]).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Frequency of a word in 36 languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#conda install -c conda-forge wordfreq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from wordfreq import word_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_frequency(\"eat\",\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_frequency(\"the\",\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = df188.split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_fre = [word_frequency(word,\"en\") for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.barplot(words, word_frequencies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "from Questgen import main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df188"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qe = main.BoolQGen()\n",
    "output = qe.predict_boolq(payload)\n",
    "pprint(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = qg.predict_shortq(payload)\n",
    "pprint(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}