gensim-doc-search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from /data/include/my-deep-learning/trump-bot-take-I/deepily/util.ipynb\n",
      "2018.07.10 12:37\n"
     ]
    }
   ],
   "source": [
    "# From: https://pypi.org/project/import-ipynb/\n",
    "import import_ipynb\n",
    "\n",
    "import deepily.util as dp_util\n",
    "from gensim import corpora, models, similarities\n",
    "from pprint import pprint  # pretty-printer\n",
    "from gensim.summarization import summarize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_FILE_PATH = \"../texts/trump-tweets.txt\"\n",
    "# INPUT_FILE_PATH = \"../texts/trump-speeches.txt\"\n",
    "\n",
    "DICTIONARY_PATH = \"output/gensim-trump-tweets.dict\"\n",
    "CORPUS_PATH = \"output/gensim-trump-tweets.mm\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dp_util.select_gpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dp_util.select_cores()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Tweets Verbatim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:37\n",
      "2018.07.10 12:37\n",
      "Time to process: [0.0062792301177978516] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = dp_util.get_time()\n",
    "documents = dp_util.load_doc_by_lines( INPUT_FILE_PATH )\n",
    "dp_util.print_time( start_time, dp_util.get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['school',\n",
      " 'shooting',\n",
      " 'texas.',\n",
      " 'early',\n",
      " 'reports',\n",
      " 'not',\n",
      " 'looking',\n",
      " 'good.',\n",
      " 'god',\n",
      " 'bless',\n",
      " 'all!']\n"
     ]
    }
   ],
   "source": [
    "# remove common words and tokenize\n",
    "stoplist = set( \"for a of the and to in\".split() )\n",
    "texts = [ [ word for word in document.lower().split() if word not in stoplist ] for document in documents ]\n",
    "\n",
    "# # remove words that appear only once\n",
    "# from collections import defaultdict\n",
    "# frequency = defaultdict(int)\n",
    "# for text in texts:\n",
    "#     for token in text:\n",
    "#         frequency[token] += 1\n",
    "\n",
    "# texts = [[token for token in text if frequency[token] > 1] for text in texts]\n",
    "pprint( texts[ 3 ] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(47359 unique tokens: ['@nikkihaley', 'again.”', 'antónio', 'around', 'conflicts']...)\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary( texts )\n",
    "dictionary.save( DICTIONARY_PATH )  # store the dictionary, for future reference\n",
    "print( dictionary )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token2id gets dict object\n",
    "# print( dictionary.token2id )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [ dictionary.doc2bow( text ) for text in texts ]\n",
    "corpora.MmCorpus.serialize( CORPUS_PATH, corpus )   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf = models.TfidfModel( corpus )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# corpus_tfidf = tfidf[ corpus ]\n",
    "# corpus_tfidf[ 3 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsi = models.LsiModel( corpus, id2word=dictionary, num_topics=300 ) # ideal number is > 300 and < 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:37\n",
      "2018.07.10 12:37\n",
      "Time to process: [3.5188889503479004] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = dp_util.get_time()\n",
    "index_lsi = similarities.MatrixSimilarity( lsi[ corpus ] )\n",
    "dp_util.print_time( start_time, dp_util.get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(171, 1), (172, 1), (7852, 1)]\n",
      "(0, 0.010897999038419407)\n"
     ]
    }
   ],
   "source": [
    "#search_doc = \"mexico build a wall\"\n",
    "search_doc = \"illegal immigrants gangs\"\n",
    "#search_doc = \"luxury condo\"\n",
    "#search_doc = \"basket of deplorables\"\n",
    "#search_doc = \"russia collusion\"\n",
    "#search_doc = \"racism in america\" # MAGA?!?\n",
    "#search_doc = \"black lives matter\" # meh!\n",
    "#search_doc = \"trump tower\" # meh!\n",
    "search_bow = dictionary.doc2bow( search_doc.lower().split() )\n",
    "print( search_bow )  \n",
    "search_lsi = lsi[ search_bow ]\n",
    "print( search_lsi[ 0 ] ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 13:24\n",
      "2018.07.10 13:24\n",
      "Time to process: [0.015276432037353516] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = dp_util.get_time()\n",
    "\n",
    "# perform a similarity query against the corpus\n",
    "similar_docs = index_lsi[ search_lsi ] \n",
    "# sort in descending order by 2nd value in tuple\n",
    "similar_docs = sorted( enumerate( similar_docs ), key=lambda item: -item[ 1 ] )\n",
    "\n",
    "dp_util.print_time( start_time, dp_util.get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(12591, 0.55641544), (12701, 0.53337717), (13671, 0.50377685), (8866, 0.49728006), (14461, 0.48256284), (12940, 0.4744817), (12038, 0.46274188)]\n",
      "[(16777, -0.16397551), (17649, -0.16397752), (16894, -0.16398136), (14267, -0.1639816), (13492, -0.16398442), (18399, -0.16398838), (14015, -0.17466487)]\n"
     ]
    }
   ],
   "source": [
    "print( similar_docs[ :7 ] )\n",
    "print( similar_docs[ -7: ] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_docs( docs ):\n",
    "    \n",
    "    for doc in docs:\n",
    "        \n",
    "        print( \"Score [%f.4] Tweet [%s]\\n\" % ( doc[ 1 ], documents[ doc[ 0 ] ].strip() ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score [0.655812.4] Tweet [The super Liberal Democrat in the Georgia Congressioal race tomorrow wants to protect criminals allow illegal immigration and raise taxes!]\n",
      "\n",
      "Score [0.576457.4] Tweet [“45 year low in illegal immigration this year.” @foxandfriends]\n",
      "\n",
      "Score [0.568625.4] Tweet [\"Anybody that believes in strong borders and stopping illegal immigration cannot vote for Marco Rubio  READ THIS: https://t.co/Tj85IsBPG8\"\"\"]\n",
      "\n",
      "Score [0.560368.4] Tweet [.@MarcoRubio is weak on illegal immigration and will allow anyone into the country.....]\n",
      "\n",
      "Score [0.522708.4] Tweet [Obama can sign an illegal executive action anytime for ObamaCare but he can’t fix the illegal loophole.]\n",
      "\n",
      "Score [0.508659.4] Tweet [The weak illegal immigration policies of the Obama Admin. allowed bad MS 13 gangs to form in cities across U.S. We are removing them fast!]\n",
      "\n",
      "Score [0.490698.4] Tweet [According to @pewresearch illegal immigrants favor Dems 8:1 http://t.co/ZN4rOIFVzA @GOP pushing amnesty. Do they have death wish]\n",
      "\n",
      "-----------------------------\n",
      "Score [-0.127684.4] Tweet [@playagaindesign  http://t.co/pdeN2jsR  Starts March 3]\n",
      "\n",
      "Score [-0.135451.4] Tweet [.@FoxNewsSunday]\n",
      "\n",
      "Score [-0.137814.4] Tweet [@TweetsFromNancy  Interesting view!]\n",
      "\n",
      "Score [-0.158704.4] Tweet [A suggestion for the dishonest media- https://t.co/zKyRwEaLmz]\n",
      "\n",
      "Score [-0.161303.4] Tweet [https://t.co/ZQ0osiFEJQ]\n",
      "\n",
      "Score [-0.167092.4] Tweet [@VintageSmitty Eric--True!]\n",
      "\n",
      "Score [-0.172832.4] Tweet [https://t.co/UM3YJ6lUiD]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_docs( similar_docs[ :7 ] )\n",
    "print( \"-----------------------------\" )\n",
    "print_docs( similar_docs[ -7: ] )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarize Top 7 Similar Docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21843, 0.80679631)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similar_docs[ 3 ]\n",
    "#summarize( similar_docs[ 3 ], ratio=0.5 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The super Liberal Democrat in the Georgia Congressioal race tomorrow wants to protect criminals allow illegal immigration and raise taxes! “45 year low in illegal immigration this year.” @foxandfriends \"Anybody that believes in strong borders and stopping illegal immigration cannot vote for Marco Rubio  READ THIS: https://t.co/Tj85IsBPG8\"\"\" .@MarcoRubio is weak on illegal immigration and will allow anyone into the country..... Obama can sign an illegal executive action anytime for ObamaCare but he can’t fix the illegal loophole. The weak illegal immigration policies of the Obama Admin. allowed bad MS 13 gangs to form in cities across U.S. We are removing them fast! According to @pewresearch illegal immigrants favor Dems 8:1 http://t.co/ZN4rOIFVzA @GOP pushing amnesty. Do they have death wish\n",
      "\n"
     ]
    }
   ],
   "source": [
    "similar_tweets = []\n",
    "\n",
    "for doc in similar_docs[ 0:7 ]:\n",
    "\n",
    "    similar_tweets.append( documents[ doc[ 0 ] ].strip() )\n",
    "    \n",
    "similar_tweets = \" \".join( similar_tweets )\n",
    "print( similar_tweets )\n",
    "print() \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The weak illegal immigration policies of the Obama Admin.\n"
     ]
    }
   ],
   "source": [
    "summary = summarize( similar_tweets, word_count=25 )# ratio=0.5 )\n",
    "print( summary )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}