word-level-rnn-w-word2vec-tweets-w-boundaries-take-II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Based on: https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/ and https://machinelearningmastery.com/how-to-develop-a-word-level-neural-language-model-in-keras/ and https://github.com/stanfordnlp/GloVe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose GPU to Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Select GPU [0 or 1]: 1\n"
     ]
    }
   ],
   "source": [
    "# From: https://github.com/keras-team/keras/issues/6031\n",
    "import os\n",
    "gpu_id = input( \"Select GPU [0 or 1]: \" )\n",
    "\n",
    "if gpu_id in [ \"0\", \"1\" ]:\n",
    "    os.environ[ \"CUDA_VISIBLE_DEVICES\" ] = gpu_id\n",
    "else:\n",
    "    print( \"Invalid GPU id.  Defaulting to '0,1'\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose CPU Cores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Share CPU cores w/ other models? [y/n]: y\n",
      "Allocating 6 cores to this notebook\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "cores = 12\n",
    "share_cores = input( \"Share CPU cores w/ other models? [y/n]: \" )\n",
    "\n",
    "if share_cores == \"y\":\n",
    "    \n",
    "    cores = int( cores / 2 )\n",
    "\n",
    "print( \"Allocating %d cores to this notebook\" % cores )\n",
    "\n",
    "# From: https://stackoverflow.com/questions/46421258/limit-number-of-cores-used-in-keras\n",
    "\n",
    "from keras import backend as K\n",
    "K.set_session(\n",
    "    K.tf.Session(\n",
    "        config=K.tf.ConfigProto(\n",
    "            intra_op_parallelism_threads=cores, inter_op_parallelism_threads=cores \n",
    "        )\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.25 11:56\n",
      "Time to process: [1] seconds\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import string\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.models import load_model\n",
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from random import randint\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "foo = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "\n",
    "print_time( 0, 1 )\n",
    "\n",
    "\n",
    "#in_filename = \"../texts/alice-in-wonderland.txt\"\n",
    "#in_filename = \"../texts/dr-zeuss-compilation.txt\"\n",
    "in_filename = \"../texts/trump-tweets.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Doc, Line by Line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.25 11:56\n",
      "2018.06.25 11:56\n",
      "Time to process: [0.01025390625] seconds\n"
     ]
    }
   ],
   "source": [
    "# http://cmdlinetips.com/2011/08/three-ways-to-read-a-text-file-line-by-line-in-python/\n",
    "def load_doc_by_line( filename ):\n",
    "    \n",
    "    # Open the file with read only permit\n",
    "    file = open( filename, \"r\" )\n",
    "    \n",
    "    # use readlines to read all lines in the file\n",
    "    # The variable \"lines\" is a list containing all lines in the file\n",
    "    lines = file.readlines()\n",
    "    \n",
    "    # close the file after reading the lines.\n",
    "    file.close()\n",
    "    \n",
    "    return lines\n",
    "\n",
    "start_time = get_time()\n",
    "tweets = load_doc_by_line( in_filename )\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Tweet Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweets [22322], total words [392403], (mean words & chars)/tweet [17.58] words, [112.10] chars\n",
      "Max/Min words [62/1] per tweet\n",
      "Max tweet: Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\n",
      "\n",
      "Min tweet: https://t.co/6VLQYAlcto\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# approximate words per tweet\n",
    "word_count = 0\n",
    "tweet_lens = 0\n",
    "max_words = 0\n",
    "max_idx = 0\n",
    "min_idx = 0\n",
    "min_words = 100\n",
    "for i, tweet in enumerate( tweets ):\n",
    "    \n",
    "    tweet_lens += len( tweet )\n",
    "    words = len( tweet.split( \" \" ) )\n",
    "    word_count += words\n",
    "    if words > max_words:\n",
    "        max_words = words\n",
    "        max_id = i\n",
    "    if words < min_words:\n",
    "        min_words = words\n",
    "        min_idx = i\n",
    "\n",
    "words_per_tweet = word_count / len( tweets )\n",
    "chars_pre_tweet = tweet_lens / len( tweets )\n",
    "\n",
    "print( \"Tweets [%d], total words [%d], (mean words & chars)/tweet [%.2f] words, [%.2f] chars\" % ( len( tweets ), word_count, words_per_tweet, chars_pre_tweet ) )\n",
    "print( \"Max/Min words [%d/%d] per tweet\" % ( max_words, min_words ) )\n",
    "print( \"Max tweet:\", tweets[ max_idx ] )\n",
    "print( \"Min tweet:\", tweets[ min_idx ] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'opentweetopen Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\\n closetweetclose'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add tweet oepn/close tags\n",
    "tweets = [ \"opentweetopen {} closetweetclose\".format( tweet ) for tweet in tweets ]\n",
    "tweets[ 0 ]          \n",
    "          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create doc from individual tweets\n",
    "doc = \" \".join( tweets )\n",
    "tweets = None\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc( filename ):\n",
    "    \n",
    "    # open the file as read only\n",
    "    file = open( filename, 'r' )\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    \n",
    "    return text\n",
    "\n",
    "# # load document\n",
    "# doc = load_doc( in_filename )\n",
    "# print( doc[ :200 ] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opentweetopen Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\n",
      " closetweetclose opentweetopen America is a Nation that believes in the power of redemption. America is a Nation that believes in second chances - and America is a Nation that believes that the best is always yet to come! #PrisonReform https://t.co/Yk5UJUYgHN\n",
      " closetweetclose opentweetopen We grieve for the terrible loss of life and send our support and love to everyone affected by this horrible attack in Texas. To the students families teachers and personnel at Santa Fe High School – we are wit\n"
     ]
    }
   ],
   "source": [
    "print( doc[ :800 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# my_punctuation = string.punctuation\n",
    "# print( type( my_punctuation ) )\n",
    "# print( my_punctuation )\n",
    "my_punctuation = '\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'\n",
    "my_punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Encoded Punctuation to Punctuation Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_dict = {}\n",
    "punctuation_dict[ \"endperiod\" ] = \".\"\n",
    "punctuation_dict[ \"endquestion\" ] = \"?\"\n",
    "punctuation_dict[ \"endexclamation\" ] = \"!\"\n",
    "punctuation_dict[ \"pausecomma\" ] = \",\"\n",
    "punctuation_dict[ \"pausecolon\" ] = \":\"\n",
    "punctuation_dict[ \"pausesemicolon\" ] = \";\"\n",
    "punctuation_dict[ \"smartquoteopen\" ] = '“'\n",
    "punctuation_dict[ \"smartquoteclose\" ] = '”'\n",
    "punctuation_dict[ \"attweetat\" ] = '@'\n",
    "punctuation_dict[ \"tweetlink\" ] = \"[link]\"\n",
    "punctuation_dict[ \"hashtweethash\" ] = '#'\n",
    "punctuation_dict[ \"opentweetopen\" ] = '[start]'\n",
    "punctuation_dict[ \"closetweetclose\" ] = '[end]'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc( doc, to_lower=True ):\n",
    "    \n",
    "    # replace '--' with a space ' '\n",
    "    doc = doc.replace( '--', ' ' )\n",
    "    # replace sentence simple sentence boundaries w/ unique token/markers\n",
    "    doc = doc.replace( '. ', ' endperiod ' )\n",
    "    doc = doc.replace( '! ', ' endexclamation ' )\n",
    "    doc = doc.replace( '? ', ' endquestion ' )\n",
    "    doc = doc.replace( ', ', ' pausecomma ' )\n",
    "    doc = doc.replace( ': ', ' pausecolon ' )\n",
    "    doc = doc.replace( '; ', ' pausesemicolon ' )\n",
    "    doc = doc.replace( '“', 'smartquoteopen ' )\n",
    "    doc = doc.replace( '”', ' smartquoteclose' )\n",
    "    doc = doc.replace( \"@ \", \" \" ) # remove trailing @'s first...\n",
    "    doc = doc.replace( \" @\", \" attweetat\" ) # ...then encode 1st char @'s\n",
    "    doc = doc.replace( \"# \", \" \" ) # remove trailing #'s first...\n",
    "    doc = doc.replace( \" #\", \" hashtweethash\" ) # ...then encode 1st char #'s\n",
    "    \n",
    "    # replace links w/ \"tweetlink\"\n",
    "    # basic regex here: https://bytes.com/topic/python/answers/741677-find-replace-hyperlinks-string\n",
    "    http_pattern = r'http[^\\s\\n\\r]+'\n",
    "    doc = re.sub( http_pattern , \"tweetlink\", doc )\n",
    "    \n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    \n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans( '', '', string.punctuation ) # will strip all .?!,:; that don't fit replace expr above.\n",
    "    #table = str.maketrans( '', '', my_punctuation )\n",
    "    tokens = [ w.translate( table ) for w in tokens ]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    if to_lower:\n",
    "        tokens = [ word for word in tokens if word.isalpha() ]\n",
    "    \n",
    "    # make lower case\n",
    "    tokens = [ word.lower() for word in tokens ] \n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Clean Doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.25 11:56\n",
      "['opentweetopen', 'just', 'met', 'with', 'un', 'secretarygeneral', 'antónio', 'guterres', 'who', 'is', 'working', 'hard', 'to', 'smartquoteopen', 'make', 'the', 'united', 'nations', 'great', 'again', 'smartquoteclose', 'when', 'the', 'un', 'does', 'more', 'to', 'solve', 'conflicts', 'around', 'the', 'world', 'it', 'means', 'the', 'us', 'endperiod', 'has', 'less', 'to', 'do', 'and', 'we', 'save', 'money', 'endperiod', 'attweetatnikkihaley', 'is', 'doing', 'a', 'fantastic', 'job', 'endexclamation', 'tweetlink', 'closetweetclose', 'opentweetopen', 'america', 'is', 'a', 'nation', 'that', 'believes', 'in', 'the', 'power', 'of', 'redemption', 'endperiod', 'america', 'is', 'a', 'nation', 'that', 'believes', 'in', 'second', 'chances', 'and', 'america', 'is', 'a', 'nation', 'that', 'believes', 'that', 'the', 'best', 'is', 'always', 'yet', 'to', 'come', 'endexclamation', 'hashtweethashprisonreform', 'tweetlink', 'closetweetclose', 'opentweetopen', 'we', 'grieve', 'for']\n",
      "Total Tokens: 443781\n",
      "Unique Tokens: 21062\n",
      "2018.06.25 11:56\n",
      "Time to process: [0.38708043098449707] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# clean document\n",
    "tokens = clean_doc( doc )\n",
    "tokens_unique = list( set( tokens ) )\n",
    "print( tokens[ :100 ] )\n",
    "print( 'Total Tokens: %d' % len( tokens ) )\n",
    "print( 'Unique Tokens: %d' % len( tokens_unique ) )\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "# 2018.06.01 10:14\n",
    "# Total Tokens: 399137\n",
    "# Unique Tokens: 21148\n",
    "# Time to process: [0.31341099739074707] seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Token Frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'antónio': 1,\n",
       " 'guterres': 1,\n",
       " 'is': 6207,\n",
       " 'just': 1334,\n",
       " 'met': 51,\n",
       " 'opentweetopen': 22322,\n",
       " 'secretarygeneral': 1,\n",
       " 'un': 37,\n",
       " 'who': 1028,\n",
       " 'with': 2401}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts = collections.Counter( tokens )\n",
    "first2pairs = { k: word_counts[ k ] for k in list( word_counts )[ :10 ] }\n",
    "first2pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opentweetopen:   22322\n",
      "closetweetclose:   22322\n",
      "the:   15056\n",
      "endperiod:   14523\n",
      "to:    9223\n",
      "a:    6911\n",
      "and:    6876\n",
      "tweetlink:    6742\n",
      "is:    6207\n",
      "of:    6100\n",
      "in:    5728\n",
      "on:    4075\n",
      "for:    4062\n",
      "i:    4061\n",
      "you:    3687\n",
      "be:    3470\n",
      "will:    3460\n",
      "endexclamation:    3153\n",
      "great:    3051\n",
      "that:    2463\n",
      "it:    2446\n",
      "at:    2413\n",
      "with:    2401\n",
      "are:    2397\n",
      "pausesemicolon:    2246\n"
     ]
    }
   ],
   "source": [
    "for word, count in word_counts.most_common( 25 ):\n",
    "    print( '%s: %7d' % ( word, count ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.25 11:56\n",
      "Total Sequences: 443730\n",
      "2018.06.25 11:56\n",
      "Time to process: [0.37273311614990234] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# organize into sequences of tokens\n",
    "sequence_len = 50 + 1\n",
    "sequences = list()\n",
    "\n",
    "for i in range( sequence_len, len( tokens ) ):\n",
    "    \n",
    "    # select sequence of tokens\n",
    "    seq = tokens[ i - sequence_len:i ]\n",
    "    \n",
    "    # convert into a line\n",
    "    line = ' '.join( seq )\n",
    "    \n",
    "    # store\n",
    "    sequences.append( line )\n",
    "    \n",
    "print( 'Total Sequences: %d' % len( sequences ) )\n",
    "print_time( start_time, get_time() )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to file, one dialog per line\n",
    "def save_doc( lines, filename ):\n",
    "    \n",
    "    data = '\\n'.join( lines )\n",
    "    file = open( filename, 'w' )\n",
    "    file.write( data )\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sequences to file\n",
    "out_filename = \"../texts/trump-tweets-sequences-02.txt\"\n",
    "save_doc( sequences, out_filename )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['opentweetopen just met with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic',\n",
       " 'just met with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job',\n",
       " 'met with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation',\n",
       " 'with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink',\n",
       " 'un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose',\n",
       " 'secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose opentweetopen',\n",
       " 'antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose opentweetopen america',\n",
       " 'guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose opentweetopen america is',\n",
       " 'who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose opentweetopen america is a',\n",
       " 'is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink closetweetclose opentweetopen america is a nation']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_filename = \"../texts/trump-tweets-sequences-02.txt\"\n",
    "#doc = load_doc( in_filename )\n",
    "lines = load_doc( in_filename ).split( '\\n' )\n",
    "lines[ 0:10 ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Words to Index Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# integer encode sequences of words\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts( lines )\n",
    "sequences = tokenizer.texts_to_sequences( lines )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51.0\n",
      "{51: 443730}\n"
     ]
    }
   ],
   "source": [
    "# iterate lists of lists, and get lens\n",
    "seq_len_sum = 0\n",
    "seq_len_dict = {}\n",
    "\n",
    "for seq in sequences:\n",
    "    \n",
    "    seq_len_sum += len( seq )\n",
    "    if len( seq ) in seq_len_dict:\n",
    "        seq_len_dict[ len( seq ) ] += 1\n",
    "    else:\n",
    "        seq_len_dict[ len( seq ) ] = 1\n",
    "        \n",
    "print( seq_len_sum / len( sequences ) ) \n",
    "print( seq_len_dict )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.int64'>\n",
      "\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "(51,)\n"
     ]
    }
   ],
   "source": [
    "print( type( sequences ) )\n",
    "print( type( sequences[ 0 ] ) )\n",
    "print( type( sequences[ 0 ][ 0 ] ) )\n",
    "print()\n",
    "print( type( sequences ) )\n",
    "print( type( sequences[ 0:1 ] ) )\n",
    "print( type( sequences[ 0:1 ][ 0 ] ) )\n",
    "print( sequences[ 0:1 ][ 0 ].shape )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "[    2    41   937    23  1241 21062 21061 21060    56     9   259   160\n",
      "     5    43    71     3   267   742    19    87    44    89     3  1241\n",
      "   212    69     5  1303  5362   540     3   141    21   998     3    51\n",
      "     4    38   572     5    59     7    28   818   154     4  7587     9\n",
      "   119     6   235]\n"
     ]
    }
   ],
   "source": [
    "print( type( sequences[ 0:1 ][ 0 ] ) )\n",
    "print( sequences[ 0:1 ][ 0 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "\n",
      "(443730, 51)\n",
      "<class 'numpy.ndarray'>\n",
      "[    2    41   937    23  1241 21062 21061 21060    56     9   259   160\n",
      "     5    43    71     3   267   742    19    87    44    89     3  1241\n",
      "   212    69     5  1303  5362   540     3   141    21   998     3    51\n",
      "     4    38   572     5    59     7    28   818   154     4  7587     9\n",
      "   119     6   235]\n"
     ]
    }
   ],
   "source": [
    "print( type( sequences ) )\n",
    "print( type( sequences[ 0:1 ] ) )\n",
    "print( type( sequences[ 0:1 ][ 0 ] ) )\n",
    "print()\n",
    "print( sequences.shape )\n",
    "print( type( sequences[ 0:1 ][ 0 ] ) )\n",
    "print( sequences[ 0:1 ][ 0 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# elegant! https://stackoverflow.com/questions/41971587/how-to-convert-predicted-sequence-back-to-text-in-keras\n",
    "sequences_to_texts = dict( map( reversed, tokenizer.word_index.items() ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'thank'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_to_texts[ 39 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "443730\n"
     ]
    }
   ],
   "source": [
    "print( len( sequences[ 0 ] ) == sequence_len )\n",
    "print( len( sequences ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21062\n",
      "<class 'dict'>\n",
      "39\n"
     ]
    }
   ],
   "source": [
    "print( len( tokenizer.word_index ) )\n",
    "print( type( tokenizer.word_index ) )\n",
    "print( tokenizer.word_index[ \"thank\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21063"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vocabulary size\n",
    "vocab_size = len( tokenizer.word_index ) + 1\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(443730, 51)\n",
      "[    2    41   937    23  1241 21062 21061 21060    56     9   259   160\n",
      "     5    43    71     3   267   742    19    87    44    89     3  1241\n",
      "   212    69     5  1303  5362   540     3   141    21   998     3    51\n",
      "     4    38   572     5    59     7    28   818   154     4  7587     9\n",
      "   119     6   235]\n",
      "<class 'numpy.ndarray'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print( type( sequences ))\n",
    "print( sequences.shape )\n",
    "\n",
    "print( sequences[ 0 ] )\n",
    "print( type( sequences[ 0 ] ) )\n",
    "\n",
    "# separate into input and output: for now it's 50 words input and 1 word output\n",
    "sequences = np.array( sequences )\n",
    "X = sequences[ :,:-1 ] # all rows, from word 0 up to, but not including, the last word\n",
    "y = sequences[ :,-1 ]  # all rows, last word only\n",
    "\n",
    "# Throws MemoryError\n",
    "# https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "#y = to_categorical( y, num_classes=vocab_size )\n",
    "\n",
    "seq_length = X.shape[ 1 ]\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Filter GloVe Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.25 11:56\n",
      "\n",
      "Loaded 13764 word vectors.\n",
      "\n",
      "Words not found 7298.\n",
      "2018.06.25 11:58\n",
      "Time to process: [112.34948635101318] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "glove = open( \"../glove/glove.6B.\" + str( embeddings_dimension ) + \"d.txt\" )\n",
    "\n",
    "for line in glove:\n",
    "    \n",
    "    values = line.split()\n",
    "    # 1st string is word...\n",
    "    word = values[ 0 ]\n",
    "    \n",
    "    if word in tokens_unique:\n",
    "        \n",
    "        # ...the rest are coefficients\n",
    "        coefs = np.asarray( values[ 1: ], dtype='float32' )\n",
    "        embeddings_index[ word ] = coefs\n",
    "        #print( \"*\", end=\"\" )\n",
    "    \n",
    "glove.close()\n",
    "print( '\\nLoaded %s word vectors.' % len( embeddings_index ) )\n",
    "print( '\\nWords not found %d.' % ( len( tokenizer.word_index ) - len( embeddings_index ) ) )\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform into Matrix That Maps Coefs by Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7298\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['closetweetclose',\n",
       " 'opentweetopen',\n",
       " 'endperiod',\n",
       " 'tweetlink',\n",
       " 'endexclamation',\n",
       " 'pausesemicolon',\n",
       " 'pausecolon',\n",
       " 'smartquoteopen',\n",
       " 'smartquoteclose',\n",
       " 'endquestion',\n",
       " 'attweetatbarackobama',\n",
       " 'attweetatfoxnews',\n",
       " 'hashtweethashmakeamericagreatagain',\n",
       " 'attweetatmittromney',\n",
       " 'attweetatfoxandfriends',\n",
       " 'attweetatapprenticenbc',\n",
       " 'hashtweethashcelebapprentice',\n",
       " 'attweetatcnn',\n",
       " 'attweetatbarackobamas',\n",
       " 'attweetatnytimes',\n",
       " 'attweetatcelebapprentice',\n",
       " 'hashtweethashmaga',\n",
       " 'hashtweethashtimetogettough',\n",
       " 'hashtweethashtrumpvlog',\n",
       " 'attweetatnbc',\n",
       " 'twitlonger',\n",
       " 'attweetatgretawire',\n",
       " 'attweetativankatrump',\n",
       " 'attweetatnewsmaxmedia',\n",
       " 'attweetatseanhannity',\n",
       " 'attweetatrealdonaldtrump',\n",
       " 'attweetatbillmaher',\n",
       " 'attweetatmacys',\n",
       " 'hashtweethashamericafirst',\n",
       " 'attweetatoreillyfactor',\n",
       " 'realdonaldtrump',\n",
       " 'hashtweethashdraintheswamp',\n",
       " 'attweetattrumpdoral',\n",
       " 'attweetatwhitehouse',\n",
       " 'arod',\n",
       " 'attweetatyankees',\n",
       " 'daca',\n",
       " 'attweetatgop',\n",
       " 'attweetatbreitbartnews',\n",
       " 'hashtweethashvotetrump',\n",
       " 'attweetaterictrump',\n",
       " 'maralago',\n",
       " 'attweetattrumptowerny',\n",
       " 'hashtweethashimwithyou',\n",
       " 'attweetatkarlrove',\n",
       " 'attweetatpiersmorgan',\n",
       " 'attweetatsquawkcnbc',\n",
       " 'hashtweethashdebate',\n",
       " 'attweetattrumpchicago',\n",
       " 'attweetatagschneiderman',\n",
       " 'attweetatalexsalmond',\n",
       " 'hashtweethashbigleaguetruth',\n",
       " 'hashtweethashcrookedhillary',\n",
       " 'attweetatmegynkelly',\n",
       " 'attweetatmissuniverse',\n",
       " 'attweetatabc',\n",
       " 'attweetatrosie',\n",
       " 'tonights',\n",
       " 'attweetatmorningjoe',\n",
       " 'attweetatmittromneys',\n",
       " 'youve',\n",
       " 'attweetattodayshow',\n",
       " 'attweetatwsj',\n",
       " 'hashtweethashfitn',\n",
       " 'attweetatamspec',\n",
       " 'attweetatwashingtonpost',\n",
       " 'attweetatdonaldjtrumpjr',\n",
       " 'hashtweethashgopdebate',\n",
       " 'attweetatvanityfair',\n",
       " 'attweetattrumpnewyork',\n",
       " 'attweetatmelaniatrump',\n",
       " 'attweetatpolitico',\n",
       " 'attweetatomarosa',\n",
       " 'attweetattrumpsoho',\n",
       " 'attweetatgreta',\n",
       " 'hashtweethashicymi',\n",
       " 'hashtweethashtbt',\n",
       " 'attweetatjebbush',\n",
       " 'attweetatcadillacchamp',\n",
       " 'attweetatdannyzuker',\n",
       " 'barackobama',\n",
       " 'mittromney',\n",
       " 'shouldnt',\n",
       " 'attweetatchucktodd',\n",
       " 'attweetatlateshow',\n",
       " 'alexsalmond',\n",
       " 'attweetatisrael',\n",
       " 'attweetatrnc',\n",
       " 'attweetatflotus',\n",
       " 'clientthe',\n",
       " 'attweetatap',\n",
       " 'attweetatwwe',\n",
       " 'attweetatthegarybusey',\n",
       " 'attweetatricksantorum',\n",
       " 'attweetatmcuban',\n",
       " 'attweetatnydailynews',\n",
       " 'attweetatcpacnews',\n",
       " 'attweetatmissusa',\n",
       " 'tahmooressi',\n",
       " 'attweetatdoralresort',\n",
       " 'hashtweethashobamacare',\n",
       " 'attweetathuffingtonpost',\n",
       " 'hashtweethashoscars',\n",
       " 'attweetatfundanything',\n",
       " 'attweetatnbcnews',\n",
       " 'attweetattrumpturnberry',\n",
       " 'attweetatnetanyahu',\n",
       " 'attweetatmsnbc',\n",
       " 'hillaryclinton',\n",
       " 'hashtweethashvpdebate',\n",
       " 'oreilly',\n",
       " 'attweetatteamcavuto',\n",
       " 'attweetatpennjillette',\n",
       " 'clientsmartquoteopen',\n",
       " 'attweetatjoanrivers',\n",
       " 'hashtweethashwwehof',\n",
       " 'attweetatnypost',\n",
       " 'ocare',\n",
       " 'icymi',\n",
       " 'attweetatmeetthepress',\n",
       " 'attweetatmikepence',\n",
       " 'hashtweethashiacaucus',\n",
       " 'attweetatlawrence',\n",
       " 'attweetattraceadkins',\n",
       " 'attweetatlordsugar',\n",
       " 'newsmaxiontv',\n",
       " 'attweetatvp',\n",
       " 'attweetatcnbc',\n",
       " 'attweetatdennisrodman',\n",
       " 'attweetatdmregister',\n",
       " 'attweetattrumpwaikiki',\n",
       " 'clienti',\n",
       " 'attweetattrumpgolfla',\n",
       " 'attweetatnymag',\n",
       " 'attweetatfoxbusiness',\n",
       " 'hashtweethashfakenews',\n",
       " 'attweetatingrahamangle',\n",
       " 'smartquoteclosetweetlink',\n",
       " 'youmakeamericagreatagain',\n",
       " 'attweetatjacknicklaus',\n",
       " 'attweetatunionleader',\n",
       " 'attweetatbarbarajwalters',\n",
       " 'hashtweethashtrump',\n",
       " 'attweetatmarklevinshow',\n",
       " 'hashtweethashasktrump',\n",
       " 'attweetattrumplasvegas',\n",
       " 'hashtweethashtrumptuesday',\n",
       " 'attweetatpaulryanvp',\n",
       " 'attweetatflgovscott',\n",
       " 'hashtweethashmagatweetlink',\n",
       " 'attweetatwashtimes',\n",
       " 'selffunding',\n",
       " 'attweetattime',\n",
       " 'attweetattrumpcharlotte',\n",
       " 'attweetatpressjournal',\n",
       " 'attweetatbretmichaels',\n",
       " 'againtweetlink',\n",
       " 'foxandfriends',\n",
       " 'attweetatcbsnews',\n",
       " 'attweetatlibertyu',\n",
       " 'hashtweethashtrumptrain',\n",
       " 'attweetatjimmyfallon',\n",
       " 'attweetatericbolling',\n",
       " 'attweetatbretbaier',\n",
       " 'attweetatgolfchannel',\n",
       " 'attweetatgstephanopoulos',\n",
       " 'attweetatforbes',\n",
       " 'attweetattrumpferrypoint',\n",
       " 'attweetatsrqrepublicans',\n",
       " 'hashtweethashmissuniverse',\n",
       " 'attweetattheviewtv',\n",
       " 'attweetatthefamilyleader',\n",
       " 'antitrump',\n",
       " 'djt',\n",
       " 'onesided',\n",
       " 'attweetatthebrodyfile',\n",
       " 'attweetatmorningmika',\n",
       " 'attweetatstjude',\n",
       " 'attweetatsarahpalinusa',\n",
       " 'attweetatkrauthammer',\n",
       " 'attweetathardballchris',\n",
       " 'attweetatariannahuff',\n",
       " 'attweetatnfl',\n",
       " 'attweetattrumptoronto',\n",
       " 'attweetatstevekingia',\n",
       " 'attweetatliljon',\n",
       " 'attweetatmacmiller',\n",
       " 'attweetattherealmarilu',\n",
       " 'clientmy',\n",
       " 'attweetatsenatemajldr',\n",
       " 'commanderinchief',\n",
       " 'ivankatrump',\n",
       " 'attweetathillaryclinton',\n",
       " 'hashtweethashrncincle',\n",
       " 'attweetattrumpscotland',\n",
       " 'attweetatgma',\n",
       " 'attweetatdrudgereport',\n",
       " 'attweetatnro',\n",
       " 'jebbush',\n",
       " 'attweetatgatewaypundit',\n",
       " 'attweetatdailycaller',\n",
       " 'attweetatthehill',\n",
       " 'attweetattrumppanama',\n",
       " 'attweetatjackwelch',\n",
       " 'attweetatcitizensunited',\n",
       " 'hashtweethashcelebrityapprentice',\n",
       " 'sotu',\n",
       " 'attweetatjoebiden',\n",
       " 'attweetatjonhuntsman',\n",
       " 'attweetatemmanuelmacron',\n",
       " 'hashtweethashlesm',\n",
       " 'hashtweethashtaxreform',\n",
       " 'hashtweethashflashbackfriday',\n",
       " 'attweetatpatriots',\n",
       " 'attweetatmarcorubio',\n",
       " 'attweetatisraelipm',\n",
       " 'attweetatfoxnewssunday',\n",
       " 'attweetatncgop',\n",
       " 'megynkelly',\n",
       " 'attweetatnewtgingrich',\n",
       " 'odonnell',\n",
       " 'attweetatandersoncooper',\n",
       " 'attweetatthisweekabc',\n",
       " 'attweetathowardstern',\n",
       " 'attweetatcnnpolitics',\n",
       " 'attweetatewerickson',\n",
       " 'attweetatforbesinspector',\n",
       " 'attweetaterictrumpfdn',\n",
       " 'attweetattrumpcollection',\n",
       " 'attweetattrumpireland',\n",
       " 'attweetatmediaite',\n",
       " 'attweetattigerwoods',\n",
       " 'mcuban',\n",
       " 'attweetatjohnrich',\n",
       " 'hashtweethashtrumpvine',\n",
       " 'blackdog',\n",
       " 'attweetathannityshow',\n",
       " 'attweetatdnc',\n",
       " 'attweetatjudgejeanine',\n",
       " 'attweetatbillygraham',\n",
       " 'hashtweethashimwithyoutweetlink',\n",
       " 'brexit',\n",
       " 'attweetatusatoday',\n",
       " 'oreillyfactor',\n",
       " 'attweetatjaketapper',\n",
       " 'attweetatglennbeck',\n",
       " 'attweetattedcruz',\n",
       " 'attweetatdavidaxelrod',\n",
       " 'attweetatcher',\n",
       " 'hashtweethashdemdebate',\n",
       " 'worldclass',\n",
       " 'androidi',\n",
       " 'hashtweethashtrumpadvice',\n",
       " 'nycs',\n",
       " 'attweetatespn',\n",
       " 'attweetatautismspeaks',\n",
       " 'attweetatvincemcmahon',\n",
       " 'attweetatarseniohall',\n",
       " 'attweetatdanscavino',\n",
       " 'attweetatmacyscom',\n",
       " 'attweetattherealkiyosaki',\n",
       " 'attweetatjayleno',\n",
       " 'thegarybusey',\n",
       " 'attweetatobamacare',\n",
       " 'attweetatloudobbs',\n",
       " 'flotus',\n",
       " 'attweetatdcexaminer',\n",
       " 'attweetatusnavy',\n",
       " 'progrowth',\n",
       " 'hashtweethashhurricaneharvey',\n",
       " 'attweetatfacethenation',\n",
       " 'hashtweethashdraintheswamptweetlink',\n",
       " 'secy',\n",
       " 'hashtweethashmakeamericagreatagaintweetlink',\n",
       " 'karlrove',\n",
       " 'jebs',\n",
       " 'hashtweethashsupertuesday',\n",
       " 'attweetatgovchristie',\n",
       " 'attweetatmailonline',\n",
       " 'hashtweethashncgopcon',\n",
       " 'clientvia',\n",
       " 'attweetatwrestlemania',\n",
       " 'fivestar',\n",
       " 'attweetatnewyorkobserver',\n",
       " 'attweetatextratv',\n",
       " 'attweetatgolfmagazine',\n",
       " 'attweetattrumpwinery',\n",
       " 'attweetattrumpto',\n",
       " 'attweetatmlb',\n",
       " 'hashtweethashmissusa',\n",
       " 'attweetatbuzzfeed',\n",
       " 'attweetatnyjets',\n",
       " 'attweetatlatoyajackson',\n",
       " 'attweetatbeaumontanthony',\n",
       " 'attweetatqvc',\n",
       " 'antbaxter',\n",
       " 'betathe',\n",
       " 'attweetatnra',\n",
       " 'attweetatdhsgov',\n",
       " 'attweetatspeakerryan',\n",
       " 'hashtweethashs',\n",
       " 'hashtweethashusa',\n",
       " 'oclock',\n",
       " 'attweetatnbcnightlynews',\n",
       " 'attweetatrealbencarson',\n",
       " 'hashtweethashperiscope',\n",
       " 'hashtweethashobamacarefail',\n",
       " 'werent',\n",
       " 'attweetatdonlemon',\n",
       " 'hashtweethashdebatenight',\n",
       " 'attweetatfallontonight',\n",
       " 'africanamerican',\n",
       " 'attweetatjoniernst',\n",
       " 'attweetatkimguilfoyle',\n",
       " 'traceadkins',\n",
       " 'hashtweethashinprimary',\n",
       " 'attweetattrumpgolf',\n",
       " 'attweetatjdickerson',\n",
       " 'hashtweethashvotetrumpsc',\n",
       " 'hashtweethashcnndebate',\n",
       " 'attweetatlimbaugh',\n",
       " 'attweetatfitsnews',\n",
       " 'attweetatfoxnewsinsider',\n",
       " 'attweetatgolfweekmag',\n",
       " 'attweetatmiamiherald',\n",
       " 'attweetatleezeldin',\n",
       " 'attweetatperduesenate',\n",
       " 'hashtweethashfreeourmarine',\n",
       " 'attweetatnotthatactor',\n",
       " 'attweetatpgachampionship',\n",
       " 'attweetatbbcnews',\n",
       " 'attweetatdavidcameron',\n",
       " 'attweetatoprah',\n",
       " 'attweetattheblaze',\n",
       " 'vattenfallgroup',\n",
       " 'melaniatrump',\n",
       " 'hashtweethashtheartofthedeal',\n",
       " 'attweetatvattenfallgroup',\n",
       " 'attweetatkatherinewebb',\n",
       " 'attweetatgiulianarancic',\n",
       " 'attweetatcabinet',\n",
       " 'theyve',\n",
       " 'attweetatmariabartiromo',\n",
       " 'africanamericans',\n",
       " 'attweetatrandpaul',\n",
       " 'attweetatbpolitics',\n",
       " 'attweetatfema',\n",
       " 'attweetatredcross',\n",
       " 'hashtweethashisis',\n",
       " 'attweetatsuperbowl',\n",
       " 'attweetatcbnnews',\n",
       " 'presidentelect',\n",
       " 'attweetatoann',\n",
       " 'attweetatnbcsnl',\n",
       " 'hashtweethashpotus',\n",
       " 'attweetatjoenbc',\n",
       " 'hashtweethashvotetrumptweetlink',\n",
       " 'hashtweethashcaucusfortrump',\n",
       " 'hashtweethashnhprimary',\n",
       " 'hashtweethashvotetrumpnh',\n",
       " 'morningjoe',\n",
       " 'frankluntz',\n",
       " 'schlonged',\n",
       " 'attweetatbusinessinsider',\n",
       " 'attweetatredstate',\n",
       " 'billmaher',\n",
       " 'keynoting',\n",
       " 'attweetatsixteenchicago',\n",
       " 'everyones',\n",
       " 'clientif',\n",
       " 'clientcongratulations',\n",
       " 'attweetattrumpgolfdc',\n",
       " 'attweetatjenniferjjacobs',\n",
       " 'attweetateonline',\n",
       " 'cohosting',\n",
       " 'hashtweethashmidastouch',\n",
       " 'mattginellagc',\n",
       " 'attweetatbuffalobills',\n",
       " 'attweetatafphq',\n",
       " 'attweetataberdeencc',\n",
       " 'attweetathbo',\n",
       " 'hashtweethashmakedclisten',\n",
       " 'attweetattoure',\n",
       " 'attweetattimtebow',\n",
       " 'attweetataol',\n",
       " 'sexter',\n",
       " 'attweetatanthonyweiner',\n",
       " 'dannyzuker',\n",
       " 'hashtweethashfundanything',\n",
       " 'attweetatlisarinna',\n",
       " 'attweetativankatrumps',\n",
       " 'attweetatmichellemalkin',\n",
       " 'attweetatgallupnews',\n",
       " 'attweetatrepweiner',\n",
       " 'attweetatdeadspin',\n",
       " 'attweetatjohnboehner',\n",
       " 'attweetatarsenioofficial',\n",
       " 'attweetatlisalampanelli',\n",
       " 'barackobamas',\n",
       " 'clientsweepstweet',\n",
       " 'clienthe',\n",
       " 'johnboehner',\n",
       " 'attweetatjustintrudeau',\n",
       " 'trumprussia',\n",
       " 'hashtweethashnoko',\n",
       " 'pelosischumer',\n",
       " 'attweetatreince',\n",
       " 'attweetatricardorossello',\n",
       " 'wthe',\n",
       " 'hashtweethashunga',\n",
       " 'deplorables',\n",
       " 'hashtweethashfake',\n",
       " 'repub',\n",
       " 'attweetatfaithandfreedom',\n",
       " 'firstever',\n",
       " 'hashtweethashsuperbowl',\n",
       " 'attweetatscottwalker',\n",
       " 'washingtonpost',\n",
       " 'crookeds',\n",
       " 'hashtweethashamericafirsttweetlink',\n",
       " 'attweetatsentedcruz',\n",
       " 'donaldtrump',\n",
       " 'attweetatbillkristol',\n",
       " 'rowanne',\n",
       " 'attweetatspecialreport',\n",
       " 'hashtweethashnyprimary',\n",
       " 'hashtweethashwiprimary',\n",
       " 'attweetatcolbertlateshow',\n",
       " 'donaldjtrumpjr',\n",
       " 'tedcruz',\n",
       " 'erictrump',\n",
       " 'attweetatrichlowry',\n",
       " 'postdebate',\n",
       " 'attweetatamazon',\n",
       " 'vanityfair',\n",
       " 'carlyfiorina',\n",
       " 'attweetatoutfrontcnn',\n",
       " 'hashtweethashtrumptower',\n",
       " 'attweetatrushlimbaugh',\n",
       " 'governorperry',\n",
       " 'attweetatturnberrybuzz',\n",
       " 'attweetatunivision',\n",
       " 'attweetattripadvisor',\n",
       " 'trumpgolfla',\n",
       " 'attweetattrumpnationalny',\n",
       " 'attweetathuffpostpol',\n",
       " 'attweetatpgatour',\n",
       " 'attweetattuohy',\n",
       " 'attweetatbostonherald',\n",
       " 'attweetatdavidbossie',\n",
       " 'attweetatjonahnro',\n",
       " 'attweetatcspan',\n",
       " 'attweetatfoxsports',\n",
       " 'welldeserved',\n",
       " 'attweetatowentew',\n",
       " 'attweetattrumprink',\n",
       " 'trumpdoral',\n",
       " 'attweetatalgemeiner',\n",
       " 'lordsugar',\n",
       " 'attweetatthomasaroberts',\n",
       " 'attweetatletterman',\n",
       " 'apprenticenbc',\n",
       " 'attweetatemilymiller',\n",
       " 'attweetatteammitch',\n",
       " 'attweetatthomtillis',\n",
       " 'pistorious',\n",
       " 'piersmorgan',\n",
       " 'katyperry',\n",
       " 'vescio',\n",
       " 'attweetatcnnmoney',\n",
       " 'attweetatmikebloomberg',\n",
       " 'attweetatsharkgregnorman',\n",
       " 'attweetatthedailyshow',\n",
       " 'attweetatlodgeatdoonbeg',\n",
       " 'attweetateminofficial',\n",
       " 'attweetataberdeenshire',\n",
       " 'attweetatrspbscotland',\n",
       " 'ivankas',\n",
       " 'obamacares',\n",
       " 'attweetatbillrancic',\n",
       " 'attweetatkatieshow',\n",
       " 'attweetatcondenastcorp',\n",
       " 'attweetattelemundo',\n",
       " 'attweetatgoangelo',\n",
       " 'attweetatamandatmiller',\n",
       " 'attweetatscotsmandotcom',\n",
       " 'hardballchris',\n",
       " 'pennjillette',\n",
       " 'attweetatbrandenroderick',\n",
       " 'attweetatdeesnider',\n",
       " 'barbarajwalters',\n",
       " 'attweetatlancearmstrong',\n",
       " 'attweetatmarleematlin',\n",
       " 'clientcelebrityapprentice',\n",
       " 'attweetatamericanowradio',\n",
       " 'newswashington',\n",
       " 'seanhannity',\n",
       " 'attweetatastros',\n",
       " 'attweetattuckercarlson',\n",
       " 'meritbased',\n",
       " 'sampp',\n",
       " 'wapo',\n",
       " 'attweetatgeraldorivera',\n",
       " 'attweetatgovmikehuckabee',\n",
       " 'attweetatusga',\n",
       " 'hashtweethashuswomensopen',\n",
       " 'attweetatgovwalker',\n",
       " 'attweetatcbs',\n",
       " 'polltrump',\n",
       " 'hashtweethashdebates',\n",
       " 'hashtweethashmagatickets',\n",
       " 'attweetatjerryjrfalwell',\n",
       " 'metweetlink',\n",
       " 'havnt',\n",
       " 'hashtweethashmakeamericasafeagain',\n",
       " 'endofmonth',\n",
       " 'melanias',\n",
       " 'attweetatserenawilliams',\n",
       " 'attweetatjasondovesq',\n",
       " 'thehill',\n",
       " 'attweetatlindseygrahamsc',\n",
       " 'ewerickson',\n",
       " 'hashtweethashscprimary',\n",
       " 'attweetatstephenathome',\n",
       " 'hashtweethashnhpolitics',\n",
       " 'bobvanderplaats',\n",
       " 'pollthank',\n",
       " 'thebrodyfile',\n",
       " 'chucktodd',\n",
       " 'attweetatreuters',\n",
       " 'attweetaterinburnett',\n",
       " 'attweetatjheil',\n",
       " 'attweetattwitternyc',\n",
       " 'rushlimbaugh',\n",
       " 'attweetatschwarzenegger',\n",
       " 'attweetatfacebook',\n",
       " 'attweetatdallasmavs',\n",
       " 'genesimmons',\n",
       " 'attweetatmets',\n",
       " 'attweetatclareoc',\n",
       " 'androidthe',\n",
       " 'clientthere',\n",
       " 'clientour',\n",
       " 'hashtweethashwhcd',\n",
       " 'attweetatnhgop',\n",
       " 'bretbaier',\n",
       " 'tmobile',\n",
       " 'attweetatjoshmcelveen',\n",
       " 'attweetatjeangeorges',\n",
       " 'attweetatdloesch',\n",
       " 'attweetatmelaniebatley',\n",
       " 'davidaxelrod',\n",
       " 'attweetatwandacarruthers',\n",
       " 'hashtweethashaskthedonald',\n",
       " 'attweetatibdeditorials',\n",
       " 'attweetatlandexpo',\n",
       " 'attweetatpeoplescompany',\n",
       " 'attweetatrobbreport',\n",
       " 'attweetatshawnjohnson',\n",
       " 'attweetatjonkarl',\n",
       " 'attweetattheeconomicclub',\n",
       " 'attweetatdamacofficial',\n",
       " 'attweetatbillcassidy',\n",
       " 'hashtweethashbluemonster',\n",
       " 'attweetatjeangeorgesnyc',\n",
       " 'attweetatsenscottbrown',\n",
       " 'attweetatlaraleayunaska',\n",
       " 'agschneiderman',\n",
       " 'attweetatstevedeaceshow',\n",
       " 'attweetatcahillforag',\n",
       " 'attweetatbriarcliffmanor',\n",
       " 'mcilroyrory',\n",
       " 'attweetatantbaxter',\n",
       " 'attweetatboonepickens',\n",
       " 'attweetattheopen',\n",
       " 'hashtweethashicebucketchallenge',\n",
       " 'fivediamond',\n",
       " 'attweetatchicagotribune',\n",
       " 'nypost',\n",
       " 'attweetatfloydmayweather',\n",
       " 'attweetatbiggovt',\n",
       " 'attweetatseniorpgachamp',\n",
       " 'attweetatmattginellagc',\n",
       " 'cliententrepreneurs',\n",
       " 'attweetattmz',\n",
       " 'attweetatbillclinton',\n",
       " 'attweetatmckaycoppins',\n",
       " 'dorals',\n",
       " 'presobama',\n",
       " 'attweetatredskins',\n",
       " 'attweetatsaintanselm',\n",
       " 'jcope',\n",
       " 'attweetatehasselbeck',\n",
       " 'attweetatdanamira',\n",
       " 'attweetatkingsthings',\n",
       " 'attweetatmileycyrus',\n",
       " 'attweetatbw',\n",
       " 'winwin',\n",
       " 'rampd',\n",
       " 'attweetatdannyzucker',\n",
       " 'attweetatralphreeds',\n",
       " 'lisarinna',\n",
       " 'dennisrodman',\n",
       " 'attweetatboeing',\n",
       " 'attweetatsalon',\n",
       " 'attweetatsternshow',\n",
       " 'attweetatzwlykins',\n",
       " 'attweetatjohnnypaulcole',\n",
       " 'clientin',\n",
       " 'clientwhen',\n",
       " 'attweetatbravoandy',\n",
       " 'hashtweethashsandy',\n",
       " 'attweetatisraels',\n",
       " 'attweetatjeep',\n",
       " 'attweetatanndromney',\n",
       " 'attweetatmedvedevrussiae',\n",
       " 'lancearmstrong',\n",
       " 'attweetatrasmussenpoll',\n",
       " 'attweetatwolfblitzercnn',\n",
       " 'attweetatteresagiudice',\n",
       " 'betawe',\n",
       " 'tweetdeckmy',\n",
       " 'attweetatthehermancain',\n",
       " 'hashtweethashtrumproast',\n",
       " 'wwwyoutubecomusermattressserta',\n",
       " 'hashtweethashnationaldayofprayer',\n",
       " 'attweetatgreggutfeld',\n",
       " 'attweetatabeshinzo',\n",
       " 'attweetatusmc',\n",
       " 'attweetatturnbullmalcolm',\n",
       " 'hashtweethashamerica',\n",
       " 'attweetatmarthamaccallum',\n",
       " 'smallbusiness',\n",
       " 'attweetatjohnkasich',\n",
       " 'attweetatgopleader',\n",
       " 'halfstaff',\n",
       " 'schumerpelosi',\n",
       " 'attweetatuscg',\n",
       " 'wouldve',\n",
       " 'attweetatgaryplayer',\n",
       " 'attweetatsenategop',\n",
       " 'attweetatnygovcuomo',\n",
       " 'hashtweethashjfkfiles',\n",
       " 'attweetatstevescalise',\n",
       " 'attweetatgovabbott',\n",
       " 'reince',\n",
       " 'danaperino',\n",
       " 'attweetatheritage',\n",
       " 'hashtweethashstanleycup',\n",
       " 'cutsreform',\n",
       " 'nbcnews',\n",
       " 'hashtweethashfema',\n",
       " 'attweetatpotus',\n",
       " 'hcare',\n",
       " 'grahamcassidy',\n",
       " 'hashtweethashlaborday',\n",
       " 'hashtweethashharvey',\n",
       " 'wifes',\n",
       " 'abcwashington',\n",
       " 'middleeast',\n",
       " 'hashtweethashkateslaw',\n",
       " 'ossoff',\n",
       " 'jobstweetlink',\n",
       " 'attweetatrepmarkmeadows',\n",
       " 'attweetatjimjordan',\n",
       " 'attweetatexxonmobil',\n",
       " 'bernies',\n",
       " 'attweetatthebigeasy',\n",
       " 'hashtweethashrepealobamacare',\n",
       " 'noontweetlink',\n",
       " 'presidenttweetlink',\n",
       " 'hashtweethashcrookedhillarytweetlink',\n",
       " 'wikileakes',\n",
       " 'attweetatsenjohnmccain',\n",
       " 'timkaine',\n",
       " 'mikepence',\n",
       " 'attweetatterrybranstad',\n",
       " 'hashtweethashhannity',\n",
       " 'milliondollar',\n",
       " 'pennsylvaniamakeamericagreatagain',\n",
       " 'hashtweethashrncincletweetlink',\n",
       " 'attweetatsinow',\n",
       " 'raisedgave',\n",
       " 'attweetatelizabethforma',\n",
       " 'attweetatthetodaysgolfer',\n",
       " 'attweetattgowdysc',\n",
       " 'soonmakeamericagreatagain',\n",
       " 'attweetatnewday',\n",
       " 'attweetatteamsters',\n",
       " 'ticketstweetlink',\n",
       " 'againinprimary',\n",
       " 'lyinted',\n",
       " 'ittweetlink',\n",
       " 'hashtweethashtrumptraintweetlink',\n",
       " 'hashtweethashnewyorkvalues',\n",
       " 'yorkmakeamericagreatagain',\n",
       " 'hashtweethashwiprimarytweetlink',\n",
       " 'attweetatrealsheriffjoe',\n",
       " 'jessebwatters',\n",
       " 'itmakeamericagreatagain',\n",
       " 'hashtweethashontherecord',\n",
       " 'hashtweethashnevadacaucus',\n",
       " 'hashtweethashvotetrumpnv',\n",
       " 'attweetatkayleighmcenany',\n",
       " 'hashtweethashteamtrump',\n",
       " 'markhalperin',\n",
       " 'findertweetlink',\n",
       " 'theyd',\n",
       " 'attweetatbobvanderplaats',\n",
       " 'attweetatcnnsitroom',\n",
       " 'attweetatwolfblitzer',\n",
       " 'hashtweethashsotu',\n",
       " 'stuartpstevens',\n",
       " 'attweetattheslystallone',\n",
       " 'jonahnro',\n",
       " 'attweetatthefix',\n",
       " 'attweetattheview',\n",
       " 'deucecrew',\n",
       " 'hashtweethashmsm',\n",
       " 'attweetatsecupp',\n",
       " 'attweetatandreatantaros',\n",
       " 'missuniverse',\n",
       " 'attweetatfox',\n",
       " 'attweetatkilmeade',\n",
       " 'superpac',\n",
       " 'attweetatcharleshurt',\n",
       " 'hashtweethashtrumptoday',\n",
       " 'qampa',\n",
       " 'attweetatbobbyjindal',\n",
       " 'governorpataki',\n",
       " 'ericbolling',\n",
       " 'brandenroderick',\n",
       " 'attweetatdanaperino',\n",
       " 'breitbartnews',\n",
       " 'attweetataswoyer',\n",
       " 'attweetatcrainschicago',\n",
       " 'attweetatpgagrandslam',\n",
       " 'hashtweethashgolf',\n",
       " 'lincolnreagan',\n",
       " 'hashtweethashtcot',\n",
       " 'attweetatswaninvestor',\n",
       " 'attweetatprnewswire',\n",
       " 'attweetatkingjames',\n",
       " 'attweetatasavagenation',\n",
       " 'attweetatalexpappas',\n",
       " 'clientwith',\n",
       " 'attweetatespngolf',\n",
       " 'attweetatedshow',\n",
       " 'attweetatstephenfhayes',\n",
       " 'serenawilliams',\n",
       " 'attweetatlasvegassun',\n",
       " 'wellearned',\n",
       " 'attweetatthescotsman',\n",
       " 'trumpsoho',\n",
       " 'hashtweethashliub',\n",
       " 'attweetatkwrcrow',\n",
       " 'attweetatwsoctv',\n",
       " 'trumpknow',\n",
       " 'attweetattrumpdorals',\n",
       " 'attweetatrwildewrites',\n",
       " 'attweetatnickjonas',\n",
       " 'attweetatexaminercom',\n",
       " 'attweetataaanews',\n",
       " 'attweetatmyrbeachonline',\n",
       " 'attweetatew',\n",
       " 'attweetatsunsentinel',\n",
       " 'attweetatworldnetdaily',\n",
       " 'obamacareromneycare',\n",
       " 'hashtweethashveteransday',\n",
       " 'attweetatnicholasballasy',\n",
       " 'flgovscott',\n",
       " 'attweetatmlauer',\n",
       " 'attweetatcntraveler',\n",
       " 'dailycaller',\n",
       " 'attweetatrustyrockets',\n",
       " 'zegarelli',\n",
       " 'graememcdowell',\n",
       " 'menie',\n",
       " 'attweetatmikeandmike',\n",
       " 'attweetatkingdommag',\n",
       " 'hashtweethashbringbackourmarine',\n",
       " 'trumpnewyork',\n",
       " 'trumplasvegas',\n",
       " 'attweetatcnsnews',\n",
       " 'floortoceiling',\n",
       " 'attweetatmissteenusa',\n",
       " 'attweetatusopen',\n",
       " 'attweetatmiamiheat',\n",
       " 'attweetatprimeministersx',\n",
       " 'attweetatnba',\n",
       " 'attweetatnyrangers',\n",
       " 'arseniohall',\n",
       " 'attweetatpressclubdc',\n",
       " 'attweetatspeakerboehner',\n",
       " 'hashtweethashnhfreedomsummit',\n",
       " 'tigerwoods',\n",
       " 'attweetattheellenshow',\n",
       " 'lassner',\n",
       " 'attweetatrobastorino',\n",
       " 'birdkilling',\n",
       " 'hoaxsters',\n",
       " 'iamstevent',\n",
       " 'attweetatnewyorkgop',\n",
       " 'attweetatjamesokeefeiii',\n",
       " 'hashtweethashsavesaeed',\n",
       " 'illconceived',\n",
       " 'attweetatsdoocy',\n",
       " 'attweetatmichaelphelps',\n",
       " 'attweetatfrankmdavisjr',\n",
       " 'attweetatnetflix',\n",
       " 'attweetatoliviaculpo',\n",
       " 'attweetatredsox',\n",
       " 'nymag',\n",
       " 'attweetatpiersmorganlive',\n",
       " 'attweetatguardian',\n",
       " 'attweetatpierpaolomonni',\n",
       " 'attweetatthedailybeast',\n",
       " 'dangerweiner',\n",
       " 'hashtweethashroadhard',\n",
       " 'attweetatarod',\n",
       " 'attweetatrealsonnynewman',\n",
       " 'arods',\n",
       " 'attweetatthejbrain',\n",
       " 'attweetatkatiecouric',\n",
       " 'therealmarilu',\n",
       " 'attweetatapple',\n",
       " 'attweetatadamscott',\n",
       " 'attweetatriggsdeb',\n",
       " 'attweetatsammartinobruno',\n",
       " 'attweetatmariamenounos',\n",
       " 'attweetatmariolopezextra',\n",
       " 'attweetattodayclicker',\n",
       " 'latoyajackson',\n",
       " 'attweetatkarlroves',\n",
       " 'celebapprentice',\n",
       " 'attweetatmisskhan',\n",
       " 'attweetataccesshollywood',\n",
       " 'attweetatmarissamayer',\n",
       " 'clientwhat',\n",
       " 'attweetatlatenightjimmy',\n",
       " 'goangelo',\n",
       " 'attweetatmoorereva',\n",
       " 'attweetateawiii',\n",
       " 'attweetatmayorgas',\n",
       " 'attweetaterolyalim',\n",
       " 'attweetatletendrelarry',\n",
       " 'clientlance',\n",
       " 'clientwe',\n",
       " 'clientit',\n",
       " 'attweetatbillmahers',\n",
       " 'attweetatuberfacts',\n",
       " 'attweetatdanabrams',\n",
       " 'attweetatwtfagain',\n",
       " 'attweetatbeardymilne',\n",
       " 'clientwhy',\n",
       " 'attweetatbbc',\n",
       " 'abfalecbaldwin',\n",
       " 'attweetatdonnydeutsch',\n",
       " 'attweetatnewyorkpost',\n",
       " 'attweetatcadillac',\n",
       " 'mittromneys',\n",
       " 'attweetatnationals',\n",
       " 'attweetatcomedycentral',\n",
       " 'paternos',\n",
       " 'clientbarackobama',\n",
       " 'hashtweethashsayfie',\n",
       " 'betai',\n",
       " 'onshoring',\n",
       " 'attweetatsarahpalinusas',\n",
       " 'attweetatricksantorums',\n",
       " 'attweetatreppaulryans',\n",
       " 'clienttrumpvlog',\n",
       " 'attweetattimetogettough',\n",
       " 'attweetatralphreed',\n",
       " 'attweetatbobbeckel',\n",
       " 'susterens',\n",
       " 'attweetatamandaknox',\n",
       " 'attweetatobama',\n",
       " 'attweetatnikkihaley',\n",
       " 'hashtweethashpeaceofficersmemorialday',\n",
       " 'hashtweethashtakebackday',\n",
       " 'attweetatarmywpfootball',\n",
       " 'councel',\n",
       " 'jimrenacci',\n",
       " 'drunkdrugged',\n",
       " 'accountlawyer',\n",
       " 'flunkie',\n",
       " 'attweetatjpnpmo',\n",
       " 'hashtweethashtaxcuts',\n",
       " 'attweetatinterior',\n",
       " 'attweetatnatlparkservice',\n",
       " 'barriersalso',\n",
       " 'hashtweethashmedalofhonorday',\n",
       " 'attweetatambjohnbolton',\n",
       " 'hashtweethashbuildthewall',\n",
       " 'rallytweetlink',\n",
       " 'attweetatrealromadowney',\n",
       " 'attweetatcpac',\n",
       " 'attweetatwestervillepd',\n",
       " 'hashtweethashnationalprayerbreakfast',\n",
       " 'attweetatcolts',\n",
       " 'attweetaticegov',\n",
       " 'jayz',\n",
       " 'strzok',\n",
       " 'attweetatsenschumer',\n",
       " 'icegov',\n",
       " 'attweetatjamiejmcintyre',\n",
       " 'russiatrump',\n",
       " 'cutreform',\n",
       " 'attweetatshopfloornam',\n",
       " 'hashtweethashhanukkah',\n",
       " 'daveweigel',\n",
       " 'attweetatusarmy',\n",
       " 'attweetatsenorrinhatch',\n",
       " 'attweetatjbanafw',\n",
       " 'attweetatvarneyco',\n",
       " 'indiv',\n",
       " 'mtgs',\n",
       " 'hashtweethashaseansummit',\n",
       " 'indopacific',\n",
       " 'antisecond',\n",
       " 'militaryveterans',\n",
       " 'hashtweethashpearlharbor',\n",
       " 'outand',\n",
       " 'comeys',\n",
       " 'billionyr',\n",
       " 'attweetatnycmayor',\n",
       " 'hashtweethashnycstrong',\n",
       " 'hashtweethashopioidepidemic',\n",
       " 'attweetattheleegreenwood',\n",
       " 'onceinageneration',\n",
       " 'attweetatantonioguterres',\n",
       " 'hashtweethashprstrong',\n",
       " 'fakenews',\n",
       " 'mediathe',\n",
       " 'todaytweetlink',\n",
       " 'hashtweethashpuertorico',\n",
       " 'attweetattbn',\n",
       " 'hashtweethashusvi',\n",
       " 'recordhigh',\n",
       " 'warmbier',\n",
       " 'hashtweethashusaatunga',\n",
       " 'attweetatusairforce',\n",
       " 'hashtweethashneverforget',\n",
       " 'attweetatamericanlegion',\n",
       " 'hashtweethashussjohnsmccain',\n",
       " 'attweetatdeptvetaffairs',\n",
       " 'attweetatnarendramodi',\n",
       " 'jobsborder',\n",
       " 'wstate',\n",
       " 'hashtweethashteamscalise',\n",
       " 'attweetatsecretaryperry',\n",
       " 'attweetatsecretaryzinke',\n",
       " 'attweetatsecpricemd',\n",
       " 'hashtweethashussarizona',\n",
       " 'attweetatemmanuelmacronthank',\n",
       " 'hashtweethashbastilleday',\n",
       " 'rdy',\n",
       " 'washtimes',\n",
       " 'shouldertoshoulder',\n",
       " 'hashtweethashpotusinpoland',\n",
       " 'hashtweethashhappyindependenceday',\n",
       " 'hashtweethashfraudnewscnn',\n",
       " 'hashtweethashvotekarenhandel',\n",
       " 'cubanamerican',\n",
       " 'attweetathousegop',\n",
       " 'covfefe',\n",
       " 'attweetatcommercegov',\n",
       " 'hashtweethashnato',\n",
       " 'attweetatpresidentruvi',\n",
       " 'hashtweethashpotusabroad',\n",
       " 'hashtweethashmothersday',\n",
       " 'rexnord',\n",
       " 'yeartweetlink',\n",
       " 'attweetatusedgov',\n",
       " 'attweetatnasas',\n",
       " 'hashtweethashcongratspeggy',\n",
       " 'attweetatainsleyearhardt',\n",
       " 'hashtweethashconfirmgorsuch',\n",
       " 'hashtweethashrepealandreplace',\n",
       " 'attweetatsnoopdogg',\n",
       " 'hashtweethashmarchforlife',\n",
       " 'miamidade',\n",
       " 'innercities',\n",
       " 'llbean',\n",
       " 'hightweetlink',\n",
       " 'placethe',\n",
       " 'carolinatweetlink',\n",
       " 'iowatweetlink',\n",
       " 'hashtweethashelectionnight',\n",
       " 'nevadatweetlink',\n",
       " 'lawtweetlink',\n",
       " 'hashtweethashwattersworld',\n",
       " 'attweetatjessebwatters',\n",
       " 'ohiotweetlink',\n",
       " ...]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros( ( vocab_size, embeddings_dimension ) )\n",
    "missing_words = []\n",
    "\n",
    "# we need this to create empty coefficients array\n",
    "dummy_shape = embeddings_index[ \"the\" ].shape\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    \n",
    "    embedding_vector = embeddings_index.get( word )\n",
    "    \n",
    "    # not all words in our token list are in the wikipedia 400K set!\n",
    "    if embedding_vector is None:\n",
    "        \n",
    "        # report and create empty coefficients array\n",
    "        missing_words.append( word )\n",
    "        embedding_vector = np.zeros( dummy_shape )\n",
    "        \n",
    "    embedding_matrix[ i ] = embedding_vector\n",
    "    \n",
    "print( len( missing_words ) )\n",
    "missing_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7299"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confirm visually that \n",
    "print( len( embedding_matrix[ 0 ] ) )\n",
    "print( sum( embedding_matrix[ 0 ] ) )\n",
    "empty_coefficients_count = 0\n",
    "\n",
    "for i in range( len( embedding_matrix ) ):\n",
    "    if sum( embedding_matrix[ i ] ) == 0:\n",
    "        empty_coefficients_count += 1\n",
    "        \n",
    "empty_coefficients_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequences[ 0 ] [    2    41   937    23  1241 21062 21061 21060    56     9   259   160\n",
      "     5    43    71     3   267   742    19    87    44    89     3  1241\n",
      "   212    69     5  1303  5362   540     3   141    21   998     3    51\n",
      "     4    38   572     5    59     7    28   818   154     4  7587     9\n",
      "   119     6   235]\n",
      "\n",
      "X[ 0:1 ] [[    2    41   937    23  1241 21062 21061 21060    56     9   259   160\n",
      "      5    43    71     3   267   742    19    87    44    89     3  1241\n",
      "    212    69     5  1303  5362   540     3   141    21   998     3    51\n",
      "      4    38   572     5    59     7    28   818   154     4  7587     9\n",
      "    119     6]]\n",
      "\n",
      "y[ 0:3 ] [235 124  18]\n",
      "X.shape (443730, 50)\n",
      "seq_length 50\n",
      "type( X ) <class 'numpy.ndarray'>\n",
      "type( y ) <class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print( \"sequences[ 0 ]\", sequences[ 0 ] )\n",
    "print()\n",
    "print( \"X[ 0:1 ]\", X[ 0:1 ] )\n",
    "print()\n",
    "print( \"y[ 0:3 ]\", y[ 0:3 ] )\n",
    "print( \"X.shape\", X.shape )\n",
    "seq_length = len( X[ 0 ] )\n",
    "print( \"seq_length\", seq_length )\n",
    "print( \"type( X )\", type( X ) )\n",
    "print( \"type( y )\", type( y ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.3\n",
      "1.4.1\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "print( keras.__version__ )\n",
    "\n",
    "import tensorflow as tf\n",
    "print( tf.__version__ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 50, 300)           6318900   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 50, 100)           160400    \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 21063)             2127363   \n",
      "=================================================================\n",
      "Total params: 8,697,163\n",
      "Trainable params: 8,697,163\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "\n",
    "# now using a pre-trained, non-trainable embedding from glove's wiki analysis\n",
    "model.add( Embedding( vocab_size, embeddings_dimension, weights=[embedding_matrix], input_length=seq_length, trainable=True ) )\n",
    "model.add( LSTM( seq_length * 2, return_sequences=True ) )\n",
    "model.add( LSTM( seq_length * 2 ) )\n",
    "model.add( Dense( seq_length * 2, activation='relu' ) )\n",
    "\n",
    "# fixed TypeError below, downgraded keras from 2.1.5 to 2.1.3: https://github.com/keras-team/keras/issues/9621\n",
    "# TypeError: softmax() got an unexpected keyword argument 'axis'\n",
    "model.add( Dense( vocab_size, activation='softmax' ) )\n",
    "\n",
    "print( model.summary() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3466.640625\n",
      "431.64396887159535\n"
     ]
    }
   ],
   "source": [
    "# calc batch size\n",
    "print( len( sequences ) / 128 )\n",
    "print( len( sequences ) / 1028 )\n",
    "# Was:\n",
    "#batch_size = 124\n",
    "batch_size = 1024\n",
    "\n",
    "# can't remember where I read that batch sizes larger than 512 cause erratic convergence patterns.\n",
    "# TODO: find that article!\n",
    "#batch_size = 512\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model? [y/n]y\n",
      "Loading model models/trump-tweets-w-links-n-ats-03.h5\n"
     ]
    }
   ],
   "source": [
    "load = input( \"Load model? [y/n]\" )\n",
    "\n",
    "if load == \"y\":\n",
    "    \n",
    "    model_name = \"models/trump-tweets-w-links-n-ats-take-II.h5\"\n",
    "    print( \"Loading model %s\" % model_name )\n",
    "    model = load_model( model_name )\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print( \"NOT loading model, using default untrained model\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.06 10:54\n",
      "Epoch 1/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.1178 - acc: 0.7552\n",
      "Epoch 2/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0890 - acc: 0.7631\n",
      "Epoch 3/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0879 - acc: 0.7630\n",
      "Epoch 4/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0866 - acc: 0.7630\n",
      "Epoch 5/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0920 - acc: 0.7614\n",
      "Epoch 6/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.0921 - acc: 0.7618\n",
      "Epoch 7/200\n",
      "443730/443730 [==============================] - 49s 109us/step - loss: 1.0802 - acc: 0.7652\n",
      "Epoch 8/200\n",
      "443730/443730 [==============================] - 49s 109us/step - loss: 1.0819 - acc: 0.7643\n",
      "Epoch 9/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0724 - acc: 0.7666\n",
      "Epoch 10/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0740 - acc: 0.7656\n",
      "Epoch 11/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0785 - acc: 0.7646\n",
      "Epoch 12/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0824 - acc: 0.7633\n",
      "Epoch 13/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0710 - acc: 0.7663\n",
      "Epoch 14/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0772 - acc: 0.7642\n",
      "Epoch 15/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0556 - acc: 0.7693\n",
      "Epoch 16/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0476 - acc: 0.7719\n",
      "Epoch 17/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0454 - acc: 0.7720\n",
      "Epoch 18/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0385 - acc: 0.7740\n",
      "Epoch 19/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0460 - acc: 0.7714\n",
      "Epoch 20/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0673 - acc: 0.7660\n",
      "Epoch 21/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0575 - acc: 0.7677\n",
      "Epoch 22/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0548 - acc: 0.7685\n",
      "Epoch 23/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0450 - acc: 0.7718\n",
      "Epoch 24/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0472 - acc: 0.7702\n",
      "Epoch 25/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0596 - acc: 0.7667\n",
      "Epoch 26/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0463 - acc: 0.7709\n",
      "Epoch 27/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.0390 - acc: 0.7726\n",
      "Epoch 28/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0390 - acc: 0.7726\n",
      "Epoch 29/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0190 - acc: 0.7774\n",
      "Epoch 30/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0089 - acc: 0.7798\n",
      "Epoch 31/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0115 - acc: 0.7793\n",
      "Epoch 32/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0170 - acc: 0.7780\n",
      "Epoch 33/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0123 - acc: 0.7786\n",
      "Epoch 34/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 1.0169 - acc: 0.7766\n",
      "Epoch 35/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 1.0295 - acc: 0.7741\n",
      "Epoch 36/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0583 - acc: 0.7661\n",
      "Epoch 37/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 1.0214 - acc: 0.7756\n",
      "Epoch 38/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.0146 - acc: 0.7776\n",
      "Epoch 39/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.0055 - acc: 0.7799\n",
      "Epoch 40/200\n",
      "443730/443730 [==============================] - 48s 109us/step - loss: 1.0033 - acc: 0.7794\n",
      "Epoch 41/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0152 - acc: 0.7771\n",
      "Epoch 42/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0082 - acc: 0.7785\n",
      "Epoch 43/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0024 - acc: 0.7802\n",
      "Epoch 44/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9892 - acc: 0.7833\n",
      "Epoch 45/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9887 - acc: 0.7838\n",
      "Epoch 46/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0102 - acc: 0.7772\n",
      "Epoch 47/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0089 - acc: 0.7773\n",
      "Epoch 48/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9915 - acc: 0.7823\n",
      "Epoch 49/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9789 - acc: 0.7859\n",
      "Epoch 50/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9911 - acc: 0.7824\n",
      "Epoch 51/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 1.0271 - acc: 0.7720\n",
      "Epoch 52/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9842 - acc: 0.7838\n",
      "Epoch 53/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9788 - acc: 0.7847\n",
      "Epoch 54/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9770 - acc: 0.7850\n",
      "Epoch 55/200\n",
      "443730/443730 [==============================] - 48s 108us/step - loss: 0.9705 - acc: 0.7867\n",
      "Epoch 56/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9775 - acc: 0.7845\n",
      "Epoch 57/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9778 - acc: 0.7845\n",
      "Epoch 58/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9838 - acc: 0.7825\n",
      "Epoch 59/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9655 - acc: 0.7878\n",
      "Epoch 60/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9827 - acc: 0.7828\n",
      "Epoch 61/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9885 - acc: 0.7811\n",
      "Epoch 62/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9598 - acc: 0.7886\n",
      "Epoch 63/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9574 - acc: 0.7888\n",
      "Epoch 64/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9477 - acc: 0.7917\n",
      "Epoch 65/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9638 - acc: 0.7875\n",
      "Epoch 66/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9700 - acc: 0.7848\n",
      "Epoch 67/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9548 - acc: 0.7893\n",
      "Epoch 68/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9579 - acc: 0.7885\n",
      "Epoch 69/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9536 - acc: 0.7890\n",
      "Epoch 70/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 1.0111 - acc: 0.7745\n",
      "Epoch 71/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9982 - acc: 0.7773\n",
      "Epoch 72/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9445 - acc: 0.7918\n",
      "Epoch 73/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9463 - acc: 0.7907\n",
      "Epoch 74/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9231 - acc: 0.7968\n",
      "Epoch 75/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9152 - acc: 0.7988\n",
      "Epoch 76/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9353 - acc: 0.7932\n",
      "Epoch 77/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9391 - acc: 0.7923\n",
      "Epoch 78/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9791 - acc: 0.7819\n",
      "Epoch 79/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9466 - acc: 0.7901\n",
      "Epoch 80/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9352 - acc: 0.7932\n",
      "Epoch 81/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 0.9390 - acc: 0.7922\n",
      "Epoch 82/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9425 - acc: 0.7914\n",
      "Epoch 83/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 0.9244 - acc: 0.7965\n",
      "Epoch 84/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9256 - acc: 0.7959\n",
      "Epoch 85/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9354 - acc: 0.7928\n",
      "Epoch 86/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9189 - acc: 0.7969\n",
      "Epoch 87/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9150 - acc: 0.7976\n",
      "Epoch 88/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9301 - acc: 0.7941\n",
      "Epoch 89/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9336 - acc: 0.7925\n",
      "Epoch 90/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9275 - acc: 0.7943\n",
      "Epoch 91/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9391 - acc: 0.7909\n",
      "Epoch 92/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9164 - acc: 0.7968\n",
      "Epoch 93/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9222 - acc: 0.7957\n",
      "Epoch 94/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9088 - acc: 0.7991\n",
      "Epoch 95/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9199 - acc: 0.7960\n",
      "Epoch 96/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9198 - acc: 0.7955\n",
      "Epoch 97/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9068 - acc: 0.7992\n",
      "Epoch 98/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8977 - acc: 0.8017\n",
      "Epoch 99/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8887 - acc: 0.8043\n",
      "Epoch 100/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9021 - acc: 0.8000\n",
      "Epoch 101/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9145 - acc: 0.7965\n",
      "Epoch 102/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9248 - acc: 0.7936\n",
      "Epoch 103/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9238 - acc: 0.7933\n",
      "Epoch 104/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9288 - acc: 0.7923\n",
      "Epoch 105/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8979 - acc: 0.8005\n",
      "Epoch 106/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8738 - acc: 0.8064\n",
      "Epoch 107/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8646 - acc: 0.8094\n",
      "Epoch 108/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 0.8712 - acc: 0.8073\n",
      "Epoch 109/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9055 - acc: 0.7981\n",
      "Epoch 110/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9125 - acc: 0.7967\n",
      "Epoch 111/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9047 - acc: 0.7981\n",
      "Epoch 112/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.9091 - acc: 0.7969\n",
      "Epoch 113/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8978 - acc: 0.8003\n",
      "Epoch 114/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8834 - acc: 0.8037\n",
      "Epoch 115/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8775 - acc: 0.8056\n",
      "Epoch 116/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8739 - acc: 0.8062\n",
      "Epoch 117/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8744 - acc: 0.8059\n",
      "Epoch 118/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8965 - acc: 0.7994\n",
      "Epoch 119/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8966 - acc: 0.7999\n",
      "Epoch 120/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8870 - acc: 0.8023\n",
      "Epoch 121/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8770 - acc: 0.8046\n",
      "Epoch 122/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8515 - acc: 0.8119\n",
      "Epoch 123/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8564 - acc: 0.8104\n",
      "Epoch 124/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8714 - acc: 0.8056\n",
      "Epoch 125/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8833 - acc: 0.8032\n",
      "Epoch 126/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8958 - acc: 0.7995\n",
      "Epoch 127/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8752 - acc: 0.8045\n",
      "Epoch 128/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8741 - acc: 0.8043\n",
      "Epoch 129/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8582 - acc: 0.8092\n",
      "Epoch 130/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8557 - acc: 0.8100\n",
      "Epoch 131/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8821 - acc: 0.8024\n",
      "Epoch 132/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8819 - acc: 0.8021\n",
      "Epoch 133/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8692 - acc: 0.8062\n",
      "Epoch 134/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8429 - acc: 0.8131\n",
      "Epoch 135/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8298 - acc: 0.8158\n",
      "Epoch 136/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8419 - acc: 0.8128\n",
      "Epoch 137/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8708 - acc: 0.8049\n",
      "Epoch 138/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8804 - acc: 0.8027\n",
      "Epoch 139/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8780 - acc: 0.8032\n",
      "Epoch 140/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8614 - acc: 0.8076\n",
      "Epoch 141/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8385 - acc: 0.8132\n",
      "Epoch 142/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8305 - acc: 0.8150\n",
      "Epoch 143/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8431 - acc: 0.8120\n",
      "Epoch 144/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8588 - acc: 0.8080\n",
      "Epoch 145/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8558 - acc: 0.8077\n",
      "Epoch 146/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8509 - acc: 0.8097\n",
      "Epoch 147/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8465 - acc: 0.8111\n",
      "Epoch 148/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8276 - acc: 0.8161\n",
      "Epoch 149/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8437 - acc: 0.8112\n",
      "Epoch 150/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8587 - acc: 0.8074\n",
      "Epoch 151/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8445 - acc: 0.8108\n",
      "Epoch 152/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8322 - acc: 0.8146\n",
      "Epoch 153/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8381 - acc: 0.8122\n",
      "Epoch 154/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8383 - acc: 0.8120\n",
      "Epoch 155/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8267 - acc: 0.8155\n",
      "Epoch 156/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8346 - acc: 0.8129\n",
      "Epoch 157/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8301 - acc: 0.8145\n",
      "Epoch 158/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8175 - acc: 0.8170\n",
      "Epoch 159/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8175 - acc: 0.8169\n",
      "Epoch 160/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8310 - acc: 0.8140\n",
      "Epoch 161/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8183 - acc: 0.8161\n",
      "Epoch 162/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8307 - acc: 0.8133\n",
      "Epoch 163/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8426 - acc: 0.8108\n",
      "Epoch 164/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8338 - acc: 0.8131\n",
      "Epoch 165/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8490 - acc: 0.8090\n",
      "Epoch 166/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8285 - acc: 0.8144\n",
      "Epoch 167/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8032 - acc: 0.8212\n",
      "Epoch 168/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8394 - acc: 0.8114\n",
      "Epoch 169/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8615 - acc: 0.8053\n",
      "Epoch 170/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8270 - acc: 0.8142\n",
      "Epoch 171/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8080 - acc: 0.8194\n",
      "Epoch 172/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7984 - acc: 0.8223\n",
      "Epoch 173/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7883 - acc: 0.8246\n",
      "Epoch 174/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8062 - acc: 0.8204\n",
      "Epoch 175/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8279 - acc: 0.8135\n",
      "Epoch 176/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8515 - acc: 0.8069\n",
      "Epoch 177/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8127 - acc: 0.8181\n",
      "Epoch 178/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7986 - acc: 0.8213\n",
      "Epoch 179/200\n",
      "443730/443730 [==============================] - 49s 111us/step - loss: 0.7980 - acc: 0.8217\n",
      "Epoch 180/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8172 - acc: 0.8165\n",
      "Epoch 181/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8115 - acc: 0.8170\n",
      "Epoch 182/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8129 - acc: 0.8177\n",
      "Epoch 183/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8101 - acc: 0.8176\n",
      "Epoch 184/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8093 - acc: 0.8181\n",
      "Epoch 185/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8011 - acc: 0.8205\n",
      "Epoch 186/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8086 - acc: 0.8181\n",
      "Epoch 187/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8055 - acc: 0.8187\n",
      "Epoch 188/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8051 - acc: 0.8194\n",
      "Epoch 189/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7908 - acc: 0.8226\n",
      "Epoch 190/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8018 - acc: 0.8202\n",
      "Epoch 191/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7877 - acc: 0.8236\n",
      "Epoch 192/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7875 - acc: 0.8241\n",
      "Epoch 193/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7814 - acc: 0.8256\n",
      "Epoch 194/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8409 - acc: 0.8097\n",
      "Epoch 195/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8422 - acc: 0.8091\n",
      "Epoch 196/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8005 - acc: 0.8201\n",
      "Epoch 197/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7777 - acc: 0.8260\n",
      "Epoch 198/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7592 - acc: 0.8309\n",
      "Epoch 199/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.7713 - acc: 0.8280\n",
      "Epoch 200/200\n",
      "443730/443730 [==============================] - 49s 110us/step - loss: 0.8085 - acc: 0.8176\n",
      "2018.06.06 13:36\n",
      "Time to process: [2.706995360387696] hours\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "# compile model\n",
    "\n",
    "# Per comment here: https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categoricalhttps://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "model.compile( loss='sparse_categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# fit model: recent version takes ~1.5 hrs for 50 epochs = ~33% accuracy\n",
    "model.fit( X, y, batch_size=batch_size, epochs=200 )\n",
    "end_time = get_time()\n",
    "print_time( start_time, end_time, interval=\"hours\" )\n",
    "\n",
    "# was 115s/epoch, before GTX 1080 card, 94s/epoch after\n",
    "# Now 81s when batch size doubled to 1,024... And it converges faster than 512 batch.  Wuh?!?\n",
    "\n",
    "# 2018.06.05 18:21\n",
    "# Time to process: [1.338104132546319] hours for 100 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the whole model to file\n",
    "model.save( \"models/trump-tweets-w-links-n-ats-take-II.h5\" )\n",
    "\n",
    "# save the tokenizer\n",
    "dump( tokenizer, open( \"tokenizers/trump-tweets-w-links-n-ats-take-II.dump\", 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( \"embeddings/trump-tweats-w-links-n-ats-take-II.glove\", 'wb' ) )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use The Model to Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_length = len( lines[ 0 ].split() ) - 1\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'”'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "punctuation_dict.get( \"smartquoteclose\", \"bar\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq( model, tokenizer, seq_length, seed_text, n_words ):\n",
    "    \n",
    "    result = list()\n",
    "    in_text = seed_text\n",
    "    yhat = [ 0.1 ]\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for _ in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        out_word = sequences_to_texts[ yhat[ 0 ] ]\n",
    "                \n",
    "        # append to input\n",
    "        in_text += ' ' + out_word\n",
    "        \n",
    "        #result.append( out_word )\n",
    "        # substitute punctuation tags for actual punctuation\n",
    "        result.append( punctuation_dict.get( out_word, out_word ) )\n",
    "        \n",
    "        if out_word == \"closetweetclose\":\n",
    "            #print( \"Tweet end detected\" )\n",
    "            break\n",
    "            \n",
    "    print( yhat )\n",
    "    return ' '.join( result )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type( sequences_to_texts[ 1 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_punctuation( doc ):\n",
    "    \n",
    "    doc = doc.replace( ' . ', '. ' )\n",
    "    doc = doc.replace( ' ! ', '! ' )\n",
    "    doc = doc.replace( ' ? ', '? ' )\n",
    "    doc = doc.replace( ' , ', ', ' )\n",
    "    doc = doc.replace( ' : ', ': ' )\n",
    "    doc = doc.replace( ' ; ', '; ' )\n",
    "    \n",
    "    doc = doc.replace( '“ ', '“' )\n",
    "    doc = doc.replace( ' ”', '”' )\n",
    "    doc = doc.replace( \"attweetat\", '@' )\n",
    "    doc = doc.replace( \"hashtweethash\", '#' )\n",
    "    \n",
    "    return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to others but know the final decision is yours” think like a champion [end] [start] along with a soaring bar of skybound gold pool deck overlooks the city of lights [link] [end] [start] an architectural landmark @trumptowerny offers sweeping panoramic views of fifth avenue [link] [end] [start] “age is... \n",
      "\n",
      "[1]\n",
      "\n",
      "\n",
      "... whatever you think your socalled art of the deal on my life. know your and work amp; very. really badly really bravelythankavet rally with the #superbowl and its time. we said and do nothing not increase our country [end]\n"
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "# substitute the seed words\n",
    "raw_text = seed_text.split( \" \" )\n",
    "\n",
    "clean_text = [ punctuation_dict.get( word, word ) for word in raw_text ]\n",
    "clean_text = ' '.join( clean_text )\n",
    "\n",
    "print( reformat_punctuation( clean_text ) + '... \\n' )\n",
    "#print( len( seed_text.split( \" \" ) ) )\n",
    "\n",
    "# generate new text\n",
    "generated = generate_seq( model, tokenizer, seq_length, seed_text, 50 )\n",
    "# print( \"... \" + generated )\n",
    "print( \"\\n\\n... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "“age is\n",
      "[1]\n",
      "... big [link] [link] [end]\n"
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "\"opentweetopen \" + my_input\n",
    "generated = generate_seq( model, tokenizer, seq_length, my_input, 50 )\n",
    "print( \"... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}