spacey-toaster.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import tqdm\n",
    "\n",
    "nlp = spacy.load( 'en_core_web_lg' )\n",
    "#nlp = spacy.load( 'en_core_web_sm' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.05 12:38\n",
      "2018.07.05 12:38\n",
      "Time to process: [0.00014901161193847656] seconds\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import datetime\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    if interval == \"minutes\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "        \n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.624673\n",
      "0.183959\n",
      "0.180288\n"
     ]
    }
   ],
   "source": [
    "print( nlp.vocab[u'dog'].similarity( nlp.vocab[u'dachshund']) )\n",
    "print( nlp.vocab[ u'trump' ].similarity( nlp.vocab[ u'president' ] ) )\n",
    "print( nlp.vocab[ u'trump' ].similarity( nlp.vocab[ u'man' ] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<method-wrapper '__len__' of spacy.vocab.Vocab object at 0x7f44119340c8>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab.__len__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "\n",
    "cosine = lambda v1, v2: np.dot( v1, v2) / ( norm( v1 ) * norm( v2 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----------------------------\n",
      "Top 7 closest results:\n",
      "dude\n",
      "dudes\n",
      "chicks\n",
      "fucking\n",
      "ass\n",
      "guy\n",
      "dick\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# Let's see if it can figure out this analogy\n",
    "# King is to man as _____ is to woman?\n",
    "# three_words = [ \"king\", \"man\", \"\", woman\" ] # queen\n",
    "# three_words = [ \"president\", \"man\", \"\", \"woman\" ] # vice-president\n",
    "# three_words = [ \"scientist\", \"man\", \"\", \"woman\" ] # researcher? cool!\n",
    "# three_words = [ \"penis\", \"man\", \"\", \"woman\" ] # vagina!\n",
    "# three_words = [ \"pussy\", \"woman\", \"\", \"man\" ] # dick!\n",
    "# three_words = [ \"slut\", \"woman\", \"\", \"man\" ] # fucked ...WUH?!?\n",
    "# three_words = [ \"whore\", \"man\", \"\", \"woman\" ] # slut!\n",
    "# three_words = [ \"cunt\", \"woman\", \"\", \"man\" ] # twat, asshole, dick\n",
    "# three_words = [ \"nigger\", \"negro\", \"\", \"caucasian\" ] # caucasians, faggot, closeup... WUH?!?\n",
    "# three_words = [ \"bitch\", \"woman\", \"\", \"man\" ] # shit, fuckin, dude, fucker\n",
    "# three_words = [ \"douchebag\", \"man\", \"\", \"woman\" ] # asshat, douche, moron, dipshit, shithead?\n",
    "# three_words = [ \"dude\", \"man\", \"\", \"woman\" ] # chick, girl, lady, bitch\n",
    "three_words = [ \"chick\", \"woman\", \"\", \"man\" ]\n",
    "\n",
    "word_1st = nlp.vocab[ three_words[ 0 ] ]\n",
    "word_2nd = nlp.vocab[ three_words[ 1 ] ]\n",
    "word_4th = nlp.vocab[ three_words[ 3 ] ]\n",
    "\n",
    "result = word_1st.vector - word_2nd.vector + word_4th.vector\n",
    "#print( result )\n",
    "\n",
    "# gather all known words, take only the lowercased versions\n",
    "allWords = list({w for w in nlp.vocab if w.has_vector and w.is_lower and w.lower_ not in three_words})\n",
    "\n",
    "# sort by similarity to the result\n",
    "allWords.sort( key=lambda w: cosine( w.vector, result ) )\n",
    "allWords.reverse()\n",
    "print(\"\\n----------------------------\\nTop 7 closest results:\")\n",
    "for word in allWords[:7]:   \n",
    "    print( word.orth_ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24880"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len( [ w for w in nlp_mine.vocab ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_tokens( corpus, target_word, threshold ):\n",
    "    \n",
    "    #counter = 0\n",
    "    candidates = []\n",
    "    for word in keys_list:\n",
    "\n",
    "        #if counter > 48:\n",
    "        #    break\n",
    "        #counter += 1\n",
    "\n",
    "        score = corpus.vocab[ target_word ].similarity( corpus.vocab[ word ] )\n",
    "        if score > threshold:\n",
    "            candidates.append( target_word + \"=\" + word + \":\" + str( score ) )\n",
    "        \n",
    "    return candidates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From: https://github.com/explosion/spaCy/issues/276\n",
    "def get_foo( word, count=10 ):\n",
    "    filtered_words = [w for w in word.vocab if w.is_lower == word.is_lower and w.prob >= -15]\n",
    "    similarity = sorted(filtered_words, key=lambda w: word.similarity(w), reverse=True)\n",
    "    return similarity[:count]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaning\n",
      "crowdfunding\n",
      "downpayment\n",
      "forgivable\n",
      "lending\n",
      "borrowers\n",
      "endowment\n",
      "repaid\n",
      "kickstarter\n",
      "seedbox\n",
      "tulpa\n",
      "lika\n",
      "snowballed\n",
      "många\n",
      "borrower\n",
      "repayment\n",
      "nonprofit\n",
      "loans\n",
      "revolving\n",
      "farmable\n",
      "philanthropy\n",
      "loan\n",
      "grants\n",
      "b.s\n",
      "subprime\n"
     ]
    }
   ],
   "source": [
    "#candidates = get_similar_tokens( nlp, \"trump\", .20 )\n",
    "#candidates = get_foo( nlp.vocab[ u'orgasm' ], 25  )\n",
    "candidates = get_foo( nlp.vocab[ u'microloan' ], 25  )\n",
    "\n",
    "for word in candidates:\n",
    "    print( word.lower_ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys_list 24498\n"
     ]
    }
   ],
   "source": [
    "embedding_keys_path = \"embeddings/trump-tweats-w-links-n-ats-take-III.keys\"\n",
    "\n",
    "with open( embedding_keys_path, 'r' ) as keys_file:\n",
    "    \n",
    "    # omit newline char: https://stackoverflow.com/questions/12330522/reading-a-file-without-newlines\n",
    "    keys_list = keys_file.read().splitlines()\n",
    "\n",
    "print( \"keys_list\", len( keys_list) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading embeddings embeddings/trump-tweats-w-links-n-ats-take-III.glove\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(24498, 300)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_path = \"embeddings/trump-tweats-w-links-n-ats-take-III.glove\"\n",
    "print( \"Loading embeddings %s\" % embedding_path )\n",
    "embedding_matrix = pickle.load( open( embedding_path, \"rb\" ) )\n",
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24498, 300)\n"
     ]
    }
   ],
   "source": [
    "from spacy.vectors import Vectors\n",
    "\n",
    "nlp_mine = spacy.blank( 'en' )\n",
    "nlp_mine.vocab.vectors = Vectors( data=embedding_matrix, keys=keys_list )\n",
    "print( nlp_mine.vocab.vectors.shape )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0845074392332\n",
      "0.117482802258\n",
      "0.255267771416\n",
      "0.359575176797\n"
     ]
    }
   ],
   "source": [
    "print( nlp_mine.vocab[ u'trump' ].similarity( nlp_mine.vocab[ u'president' ] ) )\n",
    "print( nlp_mine.vocab[ u'trump' ].similarity( nlp_mine.vocab[ u'man' ] ) )\n",
    "print( nlp_mine.vocab[ u'trump' ].similarity( nlp_mine.vocab[ u'tower' ] ) )\n",
    "print( nlp_mine.vocab[ u'trump' ].similarity( nlp_mine.vocab[ u'condo' ] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.05 12:50\n",
      "2018.07.05 12:50\n",
      "Time to process: [0.45949363708496094] seconds\n",
      "trump 1.0\n",
      "Trump 1.0\n",
      "TRUMP 1.0\n",
      "Melania 0.45651504795\n",
      "MELANIA 0.45651504795\n",
      "Casino 0.452227508974\n",
      "casino 0.452227508974\n",
      "Hilton 0.42345148615\n",
      "Trumps 0.414337546526\n",
      "TRUMPS 0.414337546526\n",
      "trumps 0.414337546526\n",
      "Ivanka 0.406093466256\n",
      "Resorts 0.399282742864\n",
      "resorts 0.399282742864\n",
      "Wynn 0.390242137461\n",
      "casinos 0.38784101066\n",
      "Mahal 0.387827881163\n",
      "Taj 0.385192201156\n",
      "Donald 0.383422199146\n",
      "DONALD 0.383422199146\n",
      "developer 0.372490767499\n",
      "condo 0.359575176797\n",
      "Vegas 0.358032071965\n",
      "VEGAS 0.358032071965\n",
      "Resort 0.349366960007\n",
      "resort 0.349366960007\n",
      "RESORT 0.349366960007\n",
      "Hotel 0.347573866331\n",
      "hotel 0.347573866331\n",
      "HOTEL 0.347573866331\n",
      "Caesars 0.340374644378\n",
      "Las 0.322768330185\n",
      "LAS 0.322768330185\n",
      "LAs 0.322768330185\n",
      "Golf 0.321984426767\n",
      "golf 0.321984426767\n",
      "hotels 0.321549088326\n",
      "Hotels 0.321549088326\n",
      "Celebrity 0.319644958539\n",
      "celebrity 0.319644958539\n",
      "CELEBRITY 0.319644958539\n",
      "condominiums 0.314406215866\n",
      "Forbes 0.307812816973\n",
      "card 0.303954080312\n",
      "Card 0.303954080312\n",
      "CARD 0.303954080312\n"
     ]
    }
   ],
   "source": [
    "def get_similar_tokens( which_nlp, target_word, threshold ):\n",
    "    \n",
    "    start_time = get_time()\n",
    "    candidates = []\n",
    "    \n",
    "    for word in keys_list:\n",
    "\n",
    "        score = which_nlp.vocab[ target_word ].similarity( which_nlp.vocab[ word ] )\n",
    "        if score > threshold:\n",
    "            candidates.append( ( word, score ) )\n",
    "    \n",
    "    # order tuples by score: https://stackoverflow.com/questions/3121979/how-to-sort-list-tuple-of-lists-tuples\n",
    "    candidates.sort( key=lambda candidate: candidate[ 1 ], reverse=True ) \n",
    "    \n",
    "    print_time( start_time, get_time() )\n",
    "    \n",
    "    return candidates\n",
    "\n",
    "candidates = get_similar_tokens( nlp_mine, \"trump\", 0.30 )\n",
    "\n",
    "for word, score in candidates:\n",
    "    print( word, score )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.05 13:01\n",
      "2018.07.05 13:01\n",
      "Time to process: [0.4554331302642822] seconds\n",
      "sexting 1.0\n",
      "Sexting 1.0\n",
      "LAWFARE 0.311345438394\n",
      "Dobias 0.311253037462\n",
      "tweeting 0.310225709636\n",
      "TWEETING 0.310225709636\n",
      "Teens 0.291672429767\n",
      "Robocall 0.288734581852\n",
      "ROBOCALL 0.288734581852\n",
      "robocall 0.288734581852\n",
      "Weiner 0.288649663682\n",
      "Razza 0.283476015574\n",
      "unpresidential 0.280662965542\n",
      "isnt 0.279528299606\n",
      "Isnt 0.279528299606\n",
      "ISNT 0.279528299606\n",
      "somethings 0.277073138006\n",
      "hashtag 0.275318835977\n",
      "3046 0.27137843937\n",
      "Lovechild 0.270264826063\n",
      "Tramonto 0.270144537406\n",
      "50000 0.269773160923\n",
      "Irans 0.266205130831\n",
      "Phn 0.265648541795\n",
      "Levins 0.265339200918\n",
      "Løkke 0.264317001079\n",
      "2403 0.263824651171\n",
      "bullying 0.262553091185\n",
      "17000 0.262050410741\n",
      "Omarosa 0.261874226171\n",
      "Damelin 0.260122833545\n",
      "Lumby 0.257776647928\n",
      "Bho 0.257736243193\n",
      "BHO 0.257736243193\n",
      "Vlog 0.25742266697\n",
      "Rager 0.255976719465\n",
      "Colene 0.255871179715\n",
      "Sletten 0.254230518538\n",
      "surveil 0.252530839795\n",
      "Trayvon 0.252441366648\n",
      "Drumstick 0.251036462737\n",
      "Kreig 0.25102915199\n",
      "Parl 0.250487967354\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( nlp_mine, \"sexting\", 0.25 )\n",
    "\n",
    "for word, score in candidates:\n",
    "    print( word, score )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.05 13:11\n",
      "2018.07.05 13:11\n",
      "Time to process: [0.4380040168762207] seconds\n",
      "luxury 1.0\n",
      "Luxury 1.0\n",
      "luxurious 0.717888515317\n",
      "Luxurious 0.717888515317\n",
      "hotels 0.600007280332\n",
      "Hotels 0.600007280332\n",
      "Hotel 0.567024141717\n",
      "hotel 0.567024141717\n",
      "HOTEL 0.567024141717\n",
      "cars 0.540415184296\n",
      "Cars 0.540415184296\n",
      "suites 0.502500365843\n",
      "Suites 0.502500365843\n",
      "condominiums 0.488497434043\n",
      "priced 0.48536386749\n",
      "expensive 0.482812929355\n",
      "Expensive 0.482812929355\n",
      "villas 0.478832750085\n",
      "brand 0.475219068391\n",
      "Brand 0.475219068391\n",
      "car 0.472071542177\n",
      "Car 0.472071542177\n",
      "Condos 0.470885191258\n",
      "boutique 0.468374915804\n",
      "Resorts 0.448167389216\n",
      "resorts 0.448167389216\n",
      "premium 0.447225841038\n",
      "PREMIUM 0.447225841038\n",
      "Premium 0.447225841038\n",
      "Mercedes 0.440655880125\n",
      "goods 0.436876808575\n",
      "Resort 0.428569483884\n",
      "resort 0.428569483884\n",
      "RESORT 0.428569483884\n",
      "jewelry 0.426608762221\n",
      "Jewelry 0.426608762221\n",
      "SUVs 0.425439477877\n",
      "shopping 0.424567213665\n",
      "Shopping 0.424567213665\n",
      "opulent 0.421503256466\n",
      "condo 0.420629538309\n",
      "dining 0.417735837687\n",
      "Dining 0.417735837687\n",
      "restaurants 0.414241306496\n",
      "amenities 0.413792985509\n",
      "Rental 0.409636011311\n",
      "afford 0.407290653169\n",
      "sleek 0.404139980589\n",
      "Cadillac 0.402158860836\n",
      "Leisure 0.397722603771\n",
      "leisure 0.397722603771\n",
      "buyers 0.396550513526\n",
      "apartment 0.396408262573\n",
      "Models 0.39579349535\n",
      "models 0.39579349535\n",
      "mansions 0.395454452957\n",
      "furniture 0.394552525933\n",
      "Furniture 0.394552525933\n",
      "residences 0.39379543886\n",
      "rooms 0.392720993662\n",
      "Rooms 0.392720993662\n",
      "affordable 0.391516880047\n",
      "Affordable 0.391516880047\n",
      "accessories 0.388518776888\n",
      "elegant 0.387544042541\n",
      "trendy 0.386712426633\n",
      "cruise 0.385896083765\n",
      "Cruise 0.385896083765\n",
      "items 0.385616211567\n",
      "Retail 0.38521889502\n",
      "retail 0.38521889502\n",
      "accommodations 0.383659718277\n",
      "automobile 0.382721948428\n",
      "stylish 0.382338375196\n",
      "clothing 0.379866475956\n",
      "fashion 0.379397681794\n",
      "Fashion 0.379397681794\n",
      "homes 0.374114823765\n",
      "Homes 0.374114823765\n",
      "vacation 0.373123879925\n",
      "Vacation 0.373123879925\n",
      "selling 0.370760714351\n",
      "Selling 0.370760714351\n",
      "Ritz 0.370741340585\n",
      "renting 0.370677257833\n",
      "chic 0.369651969287\n",
      "residential 0.369168046445\n",
      "Residential 0.369168046445\n",
      "estate 0.367680946407\n",
      "Estate 0.367680946407\n",
      "ESTATE 0.367680946407\n",
      "restaurant 0.367592888295\n",
      "Restaurant 0.367592888295\n",
      "sales 0.365894963864\n",
      "Sales 0.365894963864\n",
      "Gucci 0.3654101315\n",
      "exclusive 0.364779990775\n",
      "Exclusive 0.364779990775\n",
      "EXCLUSIVE 0.364779990775\n",
      "tourist 0.361462112792\n",
      "autos 0.361355358094\n",
      "houses 0.359922593064\n",
      "Houses 0.359922593064\n",
      "custom 0.359155504513\n",
      "lavish 0.358951177245\n",
      "yacht 0.358065845607\n",
      "wealthy 0.357549274129\n",
      "Wealthy 0.357549274129\n",
      "sport 0.356605535615\n",
      "Sport 0.356605535615\n",
      "comfort 0.356301813864\n",
      "Comfort 0.356301813864\n",
      "interiors 0.356102876964\n",
      "Interiors 0.356102876964\n",
      "buying 0.354210468449\n",
      "Buying 0.354210468449\n",
      "BUYING 0.354210468449\n",
      "newest 0.352400692047\n",
      "private 0.352329931567\n",
      "Private 0.352329931567\n",
      "PRIVATE 0.352329931567\n",
      "owners 0.351490487647\n",
      "Owners 0.351490487647\n",
      "suite 0.350695214931\n",
      "Suite 0.350695214931\n",
      "built 0.350063887122\n",
      "Built 0.350063887122\n",
      "sold 0.34938968883\n",
      "Sold 0.34938968883\n",
      "SOLD 0.34938968883\n",
      "buy 0.348878405651\n",
      "Buy 0.348878405651\n",
      "BUY 0.348878405651\n",
      "vacations 0.348615088479\n",
      "Vacations 0.348615088479\n",
      "penthouse 0.347853510742\n",
      "Penthouse 0.347853510742\n",
      "sells 0.347302598154\n",
      "sell 0.346811019605\n",
      "Sell 0.346811019605\n",
      "bought 0.346731734617\n",
      "Bought 0.346731734617\n",
      "Riviera 0.346195145111\n",
      "auto 0.346116237574\n",
      "Auto 0.346116237574\n",
      "lifestyle 0.344264512262\n",
      "Lifestyle 0.344264512262\n",
      "clothes 0.342287370877\n",
      "Luxe 0.341514268014\n",
      "spacious 0.340158300215\n",
      "mansion 0.340130109982\n",
      "apparel 0.33979021764\n",
      "cost 0.338962859142\n",
      "Cost 0.338962859142\n",
      "COST 0.338962859142\n",
      "owns 0.338316253032\n",
      "rent 0.338300745778\n",
      "Rent 0.338300745778\n",
      "business 0.338155344814\n",
      "Business 0.338155344814\n",
      "BUSINESS 0.338155344814\n",
      "Spa 0.337275438156\n",
      "spa 0.337275438156\n",
      "property 0.337164280368\n",
      "Property 0.337164280368\n",
      "sports 0.336125261607\n",
      "Sports 0.336125261607\n",
      "destination 0.336117165406\n",
      "elegance 0.335901281435\n",
      "flagship 0.335613945441\n",
      "Golf 0.333181631479\n",
      "golf 0.333181631479\n",
      "businesses 0.332161818608\n",
      "Businesses 0.332161818608\n",
      "class 0.331828915404\n",
      "Class 0.331828915404\n",
      "stores 0.331566282323\n",
      "offering 0.330498536115\n",
      "Offering 0.330498536115\n",
      "rentals 0.329959291702\n",
      "owned 0.329857103659\n",
      "OWNED 0.329857103659\n",
      "Owned 0.329857103659\n",
      "staying 0.329624326275\n",
      "Staying 0.329624326275\n",
      "cheap 0.329367663957\n",
      "Cheap 0.329367663957\n",
      "Deluxe 0.327731028138\n",
      "deluxe 0.327731028138\n",
      "item 0.327529276521\n",
      "designer 0.326783557935\n",
      "Designer 0.326783557935\n",
      "travel 0.326502108473\n",
      "Travel 0.326502108473\n",
      "TRAVEL 0.326502108473\n",
      "Room 0.326148709081\n",
      "room 0.326148709081\n",
      "ROOM 0.326148709081\n",
      "model 0.321053399793\n",
      "Model 0.321053399793\n",
      "manufactured 0.321004648819\n",
      "Celebrity 0.320740442075\n",
      "celebrity 0.320740442075\n",
      "CELEBRITY 0.320740442075\n",
      "profitable 0.320645453708\n",
      "Hospitality 0.319385549535\n",
      "Inn 0.318117300998\n",
      "owning 0.317739551254\n",
      "trips 0.317648439486\n",
      "Trips 0.317648439486\n",
      "guests 0.317642196169\n",
      "Guests 0.317642196169\n",
      "comfortable 0.317271425821\n",
      "extravagant 0.317024945028\n",
      "inexpensive 0.316844384931\n",
      "overlooking 0.316821240316\n",
      "Overlooking 0.316821240316\n",
      "holiday 0.316563751698\n",
      "Holiday 0.316563751698\n",
      "foreigners 0.315765087991\n",
      "Foreigners 0.315765087991\n",
      "products 0.314867537417\n",
      "tourists 0.314480143602\n",
      "Tourists 0.314480143602\n",
      "building 0.31435825611\n",
      "Building 0.31435825611\n",
      "glamorous 0.314272452274\n",
      "prestige 0.313929925692\n",
      "Toyota 0.313480365528\n",
      "build 0.31329740985\n",
      "Build 0.31329740985\n",
      "BUILD 0.31329740985\n",
      "commercial 0.313281825634\n",
      "watches 0.313229931938\n",
      "Motor 0.313147333903\n",
      "purchase 0.313139772967\n",
      "Purchase 0.313139772967\n",
      "sale 0.312179496531\n",
      "Sale 0.312179496531\n",
      "buses 0.310310161379\n",
      "VIP 0.309525501812\n",
      "home 0.308697239678\n",
      "Home 0.308697239678\n",
      "HOME 0.308697239678\n",
      "residence 0.308236058145\n",
      "style 0.30815925105\n",
      "Style 0.30815925105\n",
      "Catering 0.308158679074\n",
      "costing 0.30789967784\n",
      "manufacturer 0.30770182284\n",
      "elite 0.307533914283\n",
      "Elite 0.307533914283\n",
      "store 0.307368078676\n",
      "seaside 0.307288323044\n",
      "buildings 0.306451446435\n",
      "Buildings 0.306451446435\n",
      "pleasure 0.306310121136\n",
      "Pleasure 0.306310121136\n",
      "offers 0.306197121909\n",
      "Offers 0.306197121909\n",
      "taxes 0.306167892056\n",
      "Taxes 0.306167892056\n",
      "TAXES 0.306167892056\n",
      "Armani 0.305868188914\n",
      "lounge 0.305272005335\n",
      "Palace 0.305257669744\n",
      "Estates 0.304887758117\n",
      "estates 0.304887758117\n",
      "Villa 0.303005986649\n",
      "VILLA 0.303005986649\n",
      "cheapest 0.302920439824\n",
      "entertainment 0.302885700496\n",
      "Entertainment 0.302885700496\n",
      "utility 0.30134206964\n",
      "cheaper 0.301030065003\n",
      "Cheaper 0.301030065003\n",
      "CHEAPER 0.301030065003\n",
      "purchased 0.300697444646\n",
      "shop 0.300253113746\n",
      "Shop 0.300253113746\n",
      "renaissance 0.299100143754\n",
      "merchandise 0.298832297959\n",
      "enjoying 0.298632085225\n",
      "Enjoying 0.298632085225\n",
      "offer 0.298458176189\n",
      "Offer 0.298458176189\n",
      "properties 0.298301966709\n",
      "Properties 0.298301966709\n",
      "Dior 0.298244281779\n",
      "Taj 0.29824397912\n",
      "Casino 0.297882782353\n",
      "casino 0.297882782353\n",
      "vintage 0.297874699175\n",
      "Vintage 0.297874699175\n",
      "automakers 0.297587590632\n",
      "iconic 0.297525170238\n",
      "trucks 0.297513425009\n",
      "concept 0.297222092752\n",
      "rise 0.297198593418\n",
      "Rise 0.297198593418\n",
      "beauty 0.297114659849\n",
      "Beauty 0.297114659849\n",
      "BEAUTY 0.297114659849\n",
      "designers 0.296783528586\n",
      "tax 0.296496302377\n",
      "Tax 0.296496302377\n",
      "TAX 0.296496302377\n",
      "drive 0.296440933166\n",
      "Drive 0.296440933166\n",
      "Footwear 0.295648399147\n",
      "paying 0.295485558223\n",
      "Paying 0.295485558223\n",
      "Ls 0.294880531241\n",
      "Unaffordable 0.293809530897\n",
      "unaffordable 0.293809530897\n",
      "Tickets 0.293304313951\n",
      "tickets 0.293304313951\n",
      "TICKETS 0.293304313951\n",
      "import 0.292857256596\n",
      "discounts 0.292822810171\n",
      "spending 0.292330846688\n",
      "Spending 0.292330846688\n",
      "Manhattan 0.292215945797\n",
      "maker 0.292021403566\n",
      "retailers 0.290195396054\n",
      "downtown 0.289833767969\n",
      "Downtown 0.289833767969\n",
      "Waterfront 0.28965198109\n",
      "packages 0.289629513555\n",
      "price 0.288924130082\n",
      "Price 0.288924130082\n",
      "value 0.28866367827\n",
      "Value 0.28866367827\n",
      "owner 0.288377251507\n",
      "Enjoy 0.288089520942\n",
      "enjoy 0.288089520942\n",
      "ENJOY 0.288089520942\n",
      "offices 0.287776468164\n",
      "buyer 0.28722821597\n",
      "purchases 0.286874651621\n",
      "boast 0.285806394213\n",
      "perks 0.285084429696\n",
      "even 0.285007282502\n",
      "Even 0.285007282502\n",
      "EVEN 0.285007282502\n",
      "worth 0.284835400306\n",
      "Worth 0.284835400306\n",
      "Chrysler 0.284789638123\n",
      "wealth 0.284077835424\n",
      "Wealth 0.284077835424\n",
      "spend 0.283736034191\n",
      "Spend 0.283736034191\n",
      "getaway 0.282871654014\n",
      "furnished 0.282815892634\n",
      "corporate 0.282733472029\n",
      "Corporate 0.282733472029\n",
      "personal 0.282726284179\n",
      "Personal 0.282726284179\n",
      "comparable 0.282448429243\n",
      "boasts 0.281530732807\n",
      "modest 0.281052706786\n",
      "manufacturers 0.28098315099\n",
      "Manufacturers 0.28098315099\n",
      "jet 0.279503659249\n",
      "Jeep 0.279472437745\n",
      "Travelers 0.279277098073\n",
      "Mall 0.278724969022\n",
      "driven 0.278349743005\n",
      "Deco 0.278227841044\n",
      "ship 0.278204493785\n",
      "stadiums 0.278165598366\n",
      "marketed 0.277069261595\n",
      "Hotelier 0.276552244977\n",
      "hotelier 0.276552244977\n",
      "includes 0.276212019483\n",
      "glamour 0.276132298182\n",
      "gift 0.275866302244\n",
      "Gift 0.275866302244\n",
      "GIFT 0.275866302244\n",
      "New 0.275812820889\n",
      "new 0.275812820889\n",
      "NEW 0.275812820889\n",
      "parking 0.275193210691\n",
      "ships 0.27495150252\n",
      "own 0.274933928263\n",
      "Own 0.274933928263\n",
      "OWN 0.274933928263\n",
      "available 0.274552442064\n",
      "Available 0.274552442064\n",
      "casinos 0.273743313753\n",
      "forte 0.273628262194\n",
      "empty 0.273567887242\n",
      "Empty 0.273567887242\n",
      "most 0.272819905167\n",
      "Most 0.272819905167\n",
      "MOST 0.272819905167\n",
      "makes 0.272752852713\n",
      "Makes 0.272752852713\n",
      "Besides 0.272173872503\n",
      "besides 0.272173872503\n",
      "skyscrapers 0.272166893863\n",
      "Royce 0.271746170371\n",
      "segment 0.271381711109\n",
      "truck 0.271123426925\n",
      "Truck 0.271123426925\n",
      "driving 0.271063242816\n",
      "Driving 0.271063242816\n",
      "design 0.270982994002\n",
      "Design 0.270982994002\n",
      "companies 0.270917013867\n",
      "Companies 0.270917013867\n",
      "Ford 0.27081063812\n",
      "Hilton 0.270371910607\n",
      "Beach 0.27034618694\n",
      "beach 0.27034618694\n",
      "BEACH 0.27034618694\n",
      "offered 0.269873265457\n",
      "Consumer 0.269623401357\n",
      "consumer 0.269623401357\n",
      "destinations 0.269502287432\n",
      "Destinations 0.269502287432\n",
      "recreational 0.269431478387\n",
      "like 0.269388607206\n",
      "Like 0.269388607206\n",
      "LIKE 0.269388607206\n",
      "lobby 0.269380510169\n",
      "demand 0.269266740947\n",
      "Demand 0.269266740947\n",
      "prices 0.269152263218\n",
      "PRICES 0.269152263218\n",
      "Prices 0.269152263218\n",
      "chains 0.26905079875\n",
      "Dubai 0.268661207927\n",
      "such 0.268656444824\n",
      "Such 0.268656444824\n",
      "beachside 0.268632798139\n",
      "export 0.268559430137\n",
      "product 0.268136046042\n",
      "Product 0.268136046042\n",
      "Timepieces 0.267934795386\n",
      "celebrities 0.267840238301\n",
      "Celebrities 0.267840238301\n",
      "customers 0.267789474213\n",
      "CUSTOMERS 0.267789474213\n",
      "cargo 0.267340549551\n",
      "boat 0.267185531258\n",
      "renovated 0.267104646148\n",
      "Renovated 0.267104646148\n",
      "conditioned 0.267047164339\n",
      "lots 0.266883618359\n",
      "Lots 0.266883618359\n",
      "consumers 0.266541076198\n",
      "fortune 0.2660133344\n",
      "Fortune 0.2660133344\n",
      "FORTUNE 0.2660133344\n",
      "market 0.265964170038\n",
      "Market 0.265964170038\n",
      "chain 0.265624839276\n",
      "Chain 0.265624839276\n",
      "CHAIN 0.265624839276\n",
      "cater 0.264985907595\n",
      "popular 0.26497537691\n",
      "Popular 0.26497537691\n",
      "desirable 0.264908309469\n",
      "Salon 0.264117490808\n",
      "booming 0.26400952587\n",
      "BOOMING 0.26400952587\n",
      "Diesel 0.263911743427\n",
      "French 0.263660358655\n",
      "factory 0.263364659066\n",
      "including 0.262929978315\n",
      "imposing 0.2620780184\n",
      "Imposing 0.2620780184\n",
      "passengers 0.262037227201\n",
      "designed 0.261920801541\n",
      "Designed 0.261920801541\n",
      "Trump 0.261744702549\n",
      "TRUMP 0.261744702549\n",
      "trump 0.261744702549\n",
      "small 0.261440228107\n",
      "Small 0.261440228107\n",
      "addition 0.261342689494\n",
      "few 0.261304053784\n",
      "Few 0.261304053784\n",
      "FEW 0.261304053784\n",
      "lounges 0.26106640196\n",
      "multi 0.260928464541\n",
      "expense 0.260541420045\n",
      "certain 0.260540404697\n",
      "Certain 0.260540404697\n",
      "package 0.260450260236\n",
      "Breakfast 0.260251921416\n",
      "breakfast 0.260251921416\n",
      "once 0.259936130216\n",
      "Once 0.259936130216\n",
      "ONCE 0.259936130216\n",
      "Bags 0.259901198076\n",
      "Real 0.259790253536\n",
      "real 0.259790253536\n",
      "REAL 0.259790253536\n",
      "Fit 0.259544655091\n",
      "fit 0.259544655091\n",
      "gifts 0.259417106068\n",
      "Empire 0.259274872877\n",
      "empire 0.259274872877\n",
      "ranging 0.259208796078\n",
      "Makers 0.258932987536\n",
      "majestic 0.258614793843\n",
      "skyline 0.258319123399\n",
      "equipped 0.258011728422\n",
      "midtown 0.25791919758\n",
      "Midtown 0.25791919758\n",
      "Paris 0.257833983515\n",
      "foot 0.257428887354\n",
      "Foot 0.257428887354\n",
      "Accessory 0.25740819738\n",
      "food 0.25698138873\n",
      "Food 0.25698138873\n",
      "now 0.256973352587\n",
      "Now 0.256973352587\n",
      "NOW 0.256973352587\n",
      "targeted 0.256864140647\n",
      "Targeted 0.256864140647\n",
      "option 0.256788324834\n",
      "Option 0.256788324834\n",
      "complex 0.256615190661\n",
      "Complex 0.256615190661\n",
      "costs 0.256043667642\n",
      "Costs 0.256043667642\n",
      "COSTS 0.256043667642\n",
      "least 0.256023479988\n",
      "LEAST 0.256023479988\n",
      "powered 0.255661946684\n",
      "airplane 0.255648846125\n",
      "Airplane 0.255648846125\n",
      "purchasing 0.255533899605\n",
      "pay 0.255519709987\n",
      "Pay 0.255519709987\n",
      "PAY 0.255519709987\n",
      "finest 0.255364324945\n",
      "equipment 0.255349825064\n",
      "rather 0.255107560579\n",
      "Rather 0.255107560579\n",
      "more 0.255069628659\n",
      "More 0.255069628659\n",
      "MORE 0.255069628659\n",
      "Ride 0.25482829631\n",
      "ride 0.25482829631\n",
      "quality 0.254653211973\n",
      "well 0.254587751119\n",
      "Well 0.254587751119\n",
      "WELL 0.254587751119\n",
      "plus 0.254499519611\n",
      "Plus 0.254499519611\n",
      "LIMITED 0.254115132794\n",
      "Limited 0.254115132794\n",
      "limited 0.254115132794\n",
      "large 0.253946998896\n",
      "Large 0.253946998896\n",
      "LARGE 0.253946998896\n",
      "skyscraper 0.253932867108\n",
      "Skyscraper 0.253932867108\n",
      "cafe 0.253850623344\n",
      "larger 0.253684580692\n",
      "fragrances 0.253398220126\n",
      "Towers 0.253243834903\n",
      "towers 0.253243834903\n",
      "wines 0.253046003056\n",
      "Wines 0.253046003056\n",
      "gasoline 0.252922611376\n",
      "include 0.252747438937\n",
      "INCLUDE 0.252747438937\n",
      "for 0.252216213785\n",
      "For 0.252216213785\n",
      "FOR 0.252216213785\n",
      "housing 0.252144637441\n",
      "Housing 0.252144637441\n",
      "designs 0.251738725219\n",
      "space 0.251526850388\n",
      "Space 0.251526850388\n",
      "attractions 0.251517779561\n",
      "clients 0.251180019811\n",
      "Plaza 0.250951202965\n",
      "construction 0.250874359965\n",
      "Construction 0.250874359965\n",
      "watch 0.250853901266\n",
      "Watch 0.250853901266\n",
      "WATCH 0.250853901266\n",
      "Collection 0.250765777422\n",
      "collection 0.250765777422\n",
      "COLLECTION 0.250765777422\n",
      "many 0.250361409536\n",
      "Many 0.250361409536\n",
      "MANY 0.250361409536\n",
      "outlets 0.250255085315\n",
      "imports 0.250023646459\n",
      "Imports 0.250023646459\n",
      "privately 0.250007810692\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( nlp_mine, \"luxury\", 0.25 )\n",
    "\n",
    "for word, score in candidates:\n",
    "    print( word, score )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "obama=Obama:1.0\n",
      "obama=Barack:0.92547219582\n",
      "obama=US:0.416175979137\n",
      "obama=Hillary:0.649791608994\n",
      "obama=President:0.445708892591\n",
      "obama=us:0.416175979137\n",
      "obama=Clinton:0.70856035988\n",
      "obama=Romney:0.531305474516\n",
      "obama=Republicans:0.53889969238\n",
      "obama=election:0.413482947169\n",
      "obama=Washington:0.450644435657\n",
      "obama=campaign:0.493692594791\n",
      "obama=debate:0.459067934419\n",
      "obama=Democrats:0.560830458121\n",
      "obama=House:0.404474491686\n",
      "obama=speech:0.475875558041\n",
      "obama=vote:0.409736218505\n",
      "obama=Republican:0.478441961796\n",
      "obama=Bush:0.757098791234\n",
      "obama=Vote:0.409736218505\n",
      "obama=Senate:0.475979668486\n",
      "obama=Debate:0.459067934419\n",
      "obama=White:0.41686724997\n",
      "obama=Congress:0.420323951513\n",
      "obama=Gop:0.460374901582\n",
      "obama=week:0.413215873849\n",
      "obama=leaders:0.416115479053\n",
      "obama=GOP:0.460374901582\n",
      "obama=president:0.445708892591\n",
      "obama=polls:0.43500934389\n",
      "obama=saying:0.416835073225\n",
      "obama=Senator:0.538339592759\n",
      "obama=plan:0.412532910811\n",
      "obama=candidate:0.418609397269\n",
      "obama=Administration:0.530525565123\n",
      "obama=VOTE:0.409736218505\n",
      "obama=presidential:0.534961502682\n",
      "obama=Putin:0.419637820223\n",
      "obama=Us:0.416175979137\n",
      "obama=Presidential:0.534961502682\n",
      "obama=policy:0.413515201639\n",
      "obama=Election:0.413482947169\n",
      "obama=Democrat:0.445005259098\n",
      "obama=voters:0.450636273001\n",
      "obama=message:0.4152830297\n",
      "obama=Reagan:0.406017646278\n",
      "obama=administration:0.530525565123\n",
      "obama=spoke:0.43577920888\n",
      "obama=endorsed:0.445575370479\n",
      "obama=Campaign:0.493692594791\n",
      "obama=McCain:0.759076923018\n",
      "obama=promised:0.446356979375\n",
      "obama=calls:0.408045498729\n",
      "obama=Senators:0.406753690603\n",
      "obama=Week:0.413215873849\n",
      "obama=house:0.404474491686\n",
      "obama=Biden:0.594085224204\n",
      "obama=agenda:0.423145841629\n",
      "obama=Polls:0.43500934389\n",
      "obama=promise:0.411256127282\n",
      "obama=Kerry:0.614405286112\n",
      "obama=Netanyahu:0.421644894838\n",
      "obama=push:0.403160278314\n",
      "obama=delegates:0.439374611824\n",
      "obama=Democratic:0.449582956122\n",
      "obama=Congressional:0.438390155944\n",
      "obama=nomination:0.435154844785\n",
      "obama=campaigning:0.439720523225\n",
      "obama=Pelosi:0.474309023527\n",
      "obama=Speech:0.475875558041\n",
      "obama=presidency:0.407351173185\n",
      "obama=Palin:0.500803847585\n",
      "obama=promises:0.402039710173\n",
      "obama=Plan:0.412532910811\n",
      "obama=clearly:0.405468684433\n",
      "obama=remarks:0.41091163912\n",
      "obama=nominee:0.487236213428\n",
      "obama=elect:0.439524804941\n",
      "obama=Presidency:0.407351173185\n",
      "obama=stimulus:0.404641378069\n",
      "obama=Remarks:0.41091163912\n",
      "obama=Spoke:0.43577920888\n",
      "obama=Policy:0.413515201639\n",
      "obama=Leaders:0.416115479053\n",
      "obama=Huckabee:0.412586179515\n",
      "obama=pledge:0.419258739994\n",
      "obama=CLINTON:0.70856035988\n",
      "obama=Mccain:0.759076923018\n",
      "obama=campaigned:0.40272575691\n",
      "obama=PRESIDENT:0.445708892591\n",
      "obama=BUSH:0.757098791234\n",
      "obama=proposal:0.408428888953\n",
      "obama=Inauguration:0.436998114759\n",
      "obama=candidacy:0.41514649099\n",
      "obama=Voters:0.450636273001\n",
      "obama=Elect:0.439524804941\n",
      "obama=bipartisan:0.40021162344\n",
      "obama=Merkel:0.461702357489\n",
      "obama=white:0.41686724997\n",
      "obama=senator:0.538339592759\n",
      "obama=inauguration:0.436998114759\n",
      "obama=HILLARY:0.649791608994\n",
      "obama=OBAMA:1.0\n",
      "obama=Message:0.4152830297\n",
      "obama=Karzai:0.472978105432\n",
      "obama=CAMPAIGN:0.493692594791\n",
      "obama=pledged:0.437571975552\n",
      "obama=Dole:0.492414401604\n",
      "obama=Edwards:0.443933686576\n",
      "obama=DEBATE:0.459067934419\n",
      "obama=SPEECH:0.475875558041\n",
      "obama=Medvedev:0.434950517479\n",
      "obama=Calls:0.408045498729\n",
      "obama=Lawmakers:0.460834581761\n",
      "obama=HOUSE:0.404474491686\n",
      "obama=PRESIDENTIAL:0.534961502682\n",
      "obama=Agenda:0.423145841629\n",
      "obama=Push:0.403160278314\n",
      "obama=Giuliani:0.448189881141\n",
      "obama=Aides:0.519242376722\n",
      "obama=repeatedly:0.401613233904\n",
      "obama=criticism:0.404830575371\n",
      "obama=Ahmadinejad:0.442514299019\n",
      "obama=CONGRESS:0.420323951513\n",
      "obama=Pledge:0.419258739994\n",
      "obama=PLAN:0.412532910811\n",
      "obama=Saying:0.416835073225\n",
      "obama=POLICY:0.413515201639\n",
      "obama=Candidate:0.418609397269\n",
      "obama=congress:0.420323951513\n",
      "obama=lawmakers:0.460834581761\n",
      "obama=ELECTION:0.413482947169\n",
      "obama=Bipartisan:0.40021162344\n",
      "obama=suggesting:0.401521146824\n",
      "obama=Campaigning:0.439720523225\n",
      "obama=POLLS:0.43500934389\n",
      "obama=congressional:0.438390155944\n",
      "obama=Promises:0.402039710173\n",
      "obama=SAYING:0.416835073225\n",
      "obama=SENATOR:0.538339592759\n",
      "obama=Delegates:0.439374611824\n",
      "obama=Nominee:0.487236213428\n",
      "obama=senators:0.406753690603\n",
      "obama=Nomination:0.435154844785\n",
      "obama=REPUBLICAN:0.478441961796\n",
      "obama=Proposal:0.408428888953\n",
      "obama=Hopeful:0.417645567808\n",
      "obama=LEADERS:0.416115479053\n",
      "obama=Cheney:0.497137809971\n",
      "obama=AGENDA:0.423145841629\n",
      "obama=dole:0.492414401604\n",
      "obama=SENATE:0.475979668486\n",
      "obama=DEMOCRATS:0.560830458121\n",
      "obama=Clearly:0.405468684433\n",
      "obama=Criticism:0.404830575371\n",
      "obama=Advisers:0.416238778151\n",
      "obama=WEEK:0.413215873849\n",
      "obama=aides:0.519242376722\n",
      "obama=VOTERS:0.450636273001\n",
      "obama=PUTIN:0.419637820223\n",
      "obama=advisers:0.416238778151\n",
      "obama=NOMINATION:0.435154844785\n",
      "obama=Rumsfeld:0.408605269331\n",
      "obama=PLEDGE:0.419258739994\n",
      "obama=DELEGATES:0.439374611824\n",
      "obama=hopeful:0.417645567808\n",
      "obama=DEMOCRAT:0.445005259098\n",
      "obama=MCCAIN:0.759076923018\n",
      "obama=bush:0.757098791234\n",
      "obama=PRESIDENCY:0.407351173185\n",
      "obama=Candidacy:0.41514649099\n",
      "obama=ROMNEY:0.531305474516\n",
      "obama=KERRY:0.614405286112\n",
      "obama=democratic:0.449582956122\n",
      "obama=ADMINISTRATION:0.530525565123\n",
      "obama=REPUBLICANS:0.53889969238\n",
      "obama=acknowledged:0.409164811884\n",
      "obama=hoped:0.400415556826\n",
      "obama=Stimulus:0.404641378069\n",
      "obama=republicans:0.53889969238\n",
      "obama=obama:1.0\n",
      "obama=Geithner:0.47189294261\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( \"obama\", 0.40 )\n",
    "for word in candidates:\n",
    "    print( word )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clinton=that:0.436019995415\n",
      "Clinton=Obama:0.70856035988\n",
      "Clinton=has:0.401115519932\n",
      "Clinton=he:0.42292296606\n",
      "Clinton=but:0.418666834348\n",
      "Clinton=should:0.411166340496\n",
      "Clinton=Barack:0.612398962703\n",
      "Clinton=now:0.401951585428\n",
      "Clinton=Hillary:0.692514053879\n",
      "Clinton=would:0.413885481268\n",
      "Clinton=what:0.427112079879\n",
      "Clinton=He:0.42292296606\n",
      "Clinton=President:0.549198187124\n",
      "Clinton=last:0.403753168978\n",
      "Clinton=If:0.403241456909\n",
      "Clinton=him:0.417927917382\n",
      "Clinton=Clinton:1.0\n",
      "Clinton=if:0.403241456909\n",
      "Clinton=Romney:0.443267250573\n",
      "Clinton=even:0.458548130487\n",
      "Clinton=interview:0.408684957549\n",
      "Clinton=Bill:0.517069500701\n",
      "Clinton=What:0.427112079879\n",
      "Clinton=Republicans:0.612489619751\n",
      "Clinton=election:0.425275015668\n",
      "Clinton=Washington:0.524292098236\n",
      "Clinton=campaign:0.533878879935\n",
      "Clinton=debate:0.485226446482\n",
      "Clinton=Now:0.401951585428\n",
      "Clinton=Democrats:0.587225541983\n",
      "Clinton=House:0.504525878006\n",
      "Clinton=say:0.420557312752\n",
      "Clinton=speech:0.511209961967\n",
      "Clinton=vote:0.428409611158\n",
      "Clinton=support:0.429464705682\n",
      "Clinton=Republican:0.539381087996\n",
      "Clinton=Bush:0.788992254852\n",
      "Clinton=Americans:0.406313279605\n",
      "Clinton=talk:0.406239612809\n",
      "Clinton=Vote:0.428409611158\n",
      "Clinton=meeting:0.402587844156\n",
      "Clinton=Senate:0.541390270619\n",
      "Clinton=Debate:0.485226446482\n",
      "Clinton=White:0.476299111287\n",
      "Clinton=Congress:0.506511930012\n",
      "Clinton=Gop:0.548103602118\n",
      "Clinton=both:0.414944988705\n",
      "Clinton=week:0.439776199289\n",
      "Clinton=leaders:0.463031150372\n",
      "Clinton=GOP:0.548103602118\n",
      "Clinton=president:0.549198187124\n",
      "Clinton=yet:0.419992163236\n",
      "Clinton=spending:0.409227570495\n",
      "Clinton=saying:0.457327058885\n",
      "Clinton=Senator:0.520002232787\n",
      "Clinton=ago:0.405617239352\n",
      "Clinton=political:0.415973929374\n",
      "Clinton=fact:0.458291062244\n",
      "Clinton=But:0.418666834348\n",
      "Clinton=Governor:0.40322225071\n",
      "Clinton=Tuesday:0.427181202231\n",
      "Clinton=plan:0.442952692061\n",
      "Clinton=once:0.427382131002\n",
      "Clinton=supporters:0.429802365383\n",
      "Clinton=That:0.436019995415\n",
      "Clinton=George:0.471416734673\n",
      "Clinton=candidate:0.404419098184\n",
      "Clinton=Administration:0.601520551405\n",
      "Clinton=asked:0.459814962979\n",
      "Clinton=visit:0.402557796691\n",
      "Clinton=Secretary:0.41945333148\n",
      "Clinton=VOTE:0.428409611158\n",
      "Clinton=presidential:0.58428626917\n",
      "Clinton=Even:0.458548130487\n",
      "Clinton=Putin:0.431702679126\n",
      "Clinton=Presidential:0.58428626917\n",
      "Clinton=policy:0.431958956655\n",
      "Clinton=Election:0.425275015668\n",
      "Clinton=Democrat:0.431289937183\n",
      "Clinton=voters:0.461827240041\n",
      "Clinton=message:0.401477249274\n",
      "Clinton=Clintons:0.472189661198\n",
      "Clinton=himself:0.409694512191\n",
      "Clinton=question:0.42272722655\n",
      "Clinton=Thursday:0.404511256045\n",
      "Clinton=comments:0.413159167749\n",
      "Clinton=Reagan:0.558530061101\n",
      "Clinton=NOW:0.401951585428\n",
      "Clinton=Should:0.411166340496\n",
      "Clinton=administration:0.601520551405\n",
      "Clinton=met:0.405178366347\n",
      "Clinton=Carter:0.449239911863\n",
      "Clinton=Reform:0.403321507997\n",
      "Clinton=bill:0.517069500701\n",
      "Clinton=spoke:0.449217654714\n",
      "Clinton=endorsed:0.463206561196\n",
      "Clinton=reform:0.403321507997\n",
      "Clinton=effort:0.429632971645\n",
      "Clinton=Campaign:0.533878879935\n",
      "Clinton=McCain:0.644689815975\n",
      "Clinton=promised:0.47316858757\n",
      "Clinton=calls:0.417998522872\n",
      "Clinton=Last:0.403753168978\n",
      "Clinton=Senators:0.40592853698\n",
      "Clinton=Has:0.401115519932\n",
      "Clinton=Week:0.439776199289\n",
      "Clinton=recent:0.404355620818\n",
      "Clinton=Would:0.413885481268\n",
      "Clinton=house:0.504525878006\n",
      "Clinton=though:0.426896891279\n",
      "Clinton=Wednesday:0.415536770193\n",
      "Clinton=Once:0.427382131002\n",
      "Clinton=speeches:0.412290700873\n",
      "Clinton=Biden:0.430970218606\n",
      "Clinton=whether:0.446514606637\n",
      "Clinton=agenda:0.431178472307\n",
      "Clinton=Perry:0.400980568015\n",
      "Clinton=officials:0.437682910866\n",
      "Clinton=promise:0.432974336477\n",
      "Clinton=Kerry:0.540085205891\n",
      "Clinton=recently:0.403519250302\n",
      "Clinton=Netanyahu:0.505404070716\n",
      "Clinton=clear:0.415697642445\n",
      "Clinton=Presidents:0.45128879374\n",
      "Clinton=telling:0.411326950584\n",
      "Clinton=Democratic:0.491501049884\n",
      "Clinton=Congressional:0.540156422571\n",
      "Clinton=nomination:0.442913878405\n",
      "Clinton=campaigning:0.412743806099\n",
      "Clinton=Speech:0.511209961967\n",
      "Clinton=Both:0.414944988705\n",
      "Clinton=legislation:0.427961920872\n",
      "Clinton=presidency:0.443548479795\n",
      "Clinton=Newt:0.40116502087\n",
      "Clinton=allies:0.433031611934\n",
      "Clinton=efforts:0.411279381573\n",
      "Clinton=Plan:0.442952692061\n",
      "Clinton=clearly:0.432931300479\n",
      "Clinton=personally:0.403702946774\n",
      "Clinton=Yet:0.419992163236\n",
      "Clinton=remarks:0.436473428604\n",
      "Clinton=Whether:0.446514606637\n",
      "Clinton=Say:0.420557312752\n",
      "Clinton=Gingrich:0.547401814207\n",
      "Clinton=nominee:0.458380209514\n",
      "Clinton=Meeting:0.402587844156\n",
      "Clinton=Presidency:0.443548479795\n",
      "Clinton=Visit:0.402557796691\n",
      "Clinton=Kennedy:0.425390361601\n",
      "Clinton=Political:0.415973929374\n",
      "Clinton=Remarks:0.436473428604\n",
      "Clinton=Spoke:0.449217654714\n",
      "Clinton=Fact:0.458291062244\n",
      "Clinton=Interview:0.408684957549\n",
      "Clinton=Policy:0.431958956655\n",
      "Clinton=talked:0.432887962702\n",
      "Clinton=Leaders:0.463031150372\n",
      "Clinton=THAT:0.436019995415\n",
      "Clinton=Him:0.417927917382\n",
      "Clinton=criticized:0.415127648971\n",
      "Clinton=pledge:0.439916804207\n",
      "Clinton=likely:0.420275363614\n",
      "Clinton=CLINTON:1.0\n",
      "Clinton=Mccain:0.644689815975\n",
      "Clinton=Support:0.429464705682\n",
      "Clinton=indeed:0.45419252539\n",
      "Clinton=PRESIDENT:0.549198187124\n",
      "Clinton=Met:0.405178366347\n",
      "Clinton=referring:0.424341428547\n",
      "Clinton=BUSH:0.788992254852\n",
      "Clinton=governor:0.40322225071\n",
      "Clinton=Talk:0.406239612809\n",
      "Clinton=HAS:0.401115519932\n",
      "Clinton=IF:0.403241456909\n",
      "Clinton=proposal:0.452301532577\n",
      "Clinton=Capitol:0.428811055301\n",
      "Clinton=adviser:0.440568195162\n",
      "Clinton=candidacy:0.411260862179\n",
      "Clinton=publicly:0.400917342808\n",
      "Clinton=Spending:0.409227570495\n",
      "Clinton=WHAT:0.427112079879\n",
      "Clinton=Voters:0.461827240041\n",
      "Clinton=presidents:0.45128879374\n",
      "Clinton=bipartisan:0.430690366012\n",
      "Clinton=white:0.476299111287\n",
      "Clinton=senator:0.520002232787\n",
      "Clinton=HE:0.42292296606\n",
      "Clinton=HILLARY:0.692514053879\n",
      "Clinton=neither:0.421009307762\n",
      "Clinton=TALK:0.406239612809\n",
      "Clinton=Personally:0.403702946774\n",
      "Clinton=OBAMA:0.70856035988\n",
      "Clinton=Message:0.401477249274\n",
      "Clinton=CAMPAIGN:0.533878879935\n",
      "Clinton=pledged:0.420328992953\n",
      "Clinton=WOULD:0.413885481268\n",
      "Clinton=Dole:0.694552344293\n",
      "Clinton=REFORM:0.403321507997\n",
      "Clinton=SHOULD:0.411166340496\n",
      "Clinton=Edwards:0.428121510206\n",
      "Clinton=FACT:0.458291062244\n",
      "Clinton=DEBATE:0.485226446482\n",
      "Clinton=SPEECH:0.511209961967\n",
      "Clinton=Pataki:0.456127869911\n",
      "Clinton=Blair:0.471578456195\n",
      "Clinton=Calls:0.417998522872\n",
      "Clinton=Lawmakers:0.478322245028\n",
      "Clinton=HOUSE:0.504525878006\n",
      "Clinton=veto:0.402021219567\n",
      "Clinton=PRESIDENTIAL:0.58428626917\n",
      "Clinton=Agenda:0.431178472307\n",
      "Clinton=BILL:0.517069500701\n",
      "Clinton=LAST:0.403753168978\n",
      "Clinton=Powell:0.462433239453\n",
      "Clinton=AMERICANS:0.406313279605\n",
      "Clinton=SUPPORT:0.429464705682\n",
      "Clinton=insisted:0.411315831974\n",
      "Clinton=Giuliani:0.53968380403\n",
      "Clinton=Aides:0.669289576086\n",
      "Clinton=urging:0.413442591708\n",
      "Clinton=Neither:0.421009307762\n",
      "Clinton=repeatedly:0.453516590412\n",
      "Clinton=criticism:0.412549752042\n",
      "Clinton=secretary:0.41945333148\n",
      "Clinton=CONGRESS:0.506511930012\n",
      "Clinton=Legislation:0.427961920872\n",
      "Clinton=sought:0.456882016573\n",
      "Clinton=Starr:0.411797296235\n",
      "Clinton=Question:0.42272722655\n",
      "Clinton=Pledge:0.439916804207\n",
      "Clinton=pressed:0.420844717541\n",
      "Clinton=Officials:0.437682910866\n",
      "Clinton=Asked:0.459814962979\n",
      "Clinton=PLAN:0.442952692061\n",
      "Clinton=Supporters:0.429802365383\n",
      "Clinton=Saying:0.457327058885\n",
      "Clinton=CLINTONS:0.472189661198\n",
      "Clinton=POLICY:0.431958956655\n",
      "Clinton=Candidate:0.404419098184\n",
      "Clinton=BUT:0.418666834348\n",
      "Clinton=congress:0.506511930012\n",
      "Clinton=Clear:0.415697642445\n",
      "Clinton=lawmakers:0.478322245028\n",
      "Clinton=ELECTION:0.425275015668\n",
      "Clinton=Bipartisan:0.430690366012\n",
      "Clinton=suggesting:0.416066088275\n",
      "Clinton=Nixon:0.467310971177\n",
      "Clinton=Campaigning:0.412743806099\n",
      "Clinton=SUPPORTERS:0.429802365383\n",
      "Clinton=congressional:0.540156422571\n",
      "Clinton=Adviser:0.440568195162\n",
      "Clinton=SAYING:0.457327058885\n",
      "Clinton=SAY:0.420557312752\n",
      "Clinton=Likely:0.420275363614\n",
      "Clinton=SENATOR:0.520002232787\n",
      "Clinton=Nominee:0.458380209514\n",
      "Clinton=senators:0.40592853698\n",
      "Clinton=Nomination:0.442913878405\n",
      "Clinton=REPUBLICAN:0.539381087996\n",
      "Clinton=suggested:0.464174825873\n",
      "Clinton=Proposal:0.452301532577\n",
      "Clinton=Comments:0.413159167749\n",
      "Clinton=LEADERS:0.463031150372\n",
      "Clinton=Himself:0.409694512191\n",
      "Clinton=ONCE:0.427382131002\n",
      "Clinton=Cheney:0.556105809251\n",
      "Clinton=MEETING:0.402587844156\n",
      "Clinton=Talked:0.432887962702\n",
      "Clinton=AGENDA:0.431178472307\n",
      "Clinton=VETO:0.402021219567\n",
      "Clinton=dole:0.694552344293\n",
      "Clinton=Speeches:0.412290700873\n",
      "Clinton=SENATE:0.541390270619\n",
      "Clinton=DEMOCRATS:0.587225541983\n",
      "Clinton=PRESIDENTS:0.45128879374\n",
      "Clinton=Recent:0.404355620818\n",
      "Clinton=POLITICAL:0.415973929374\n",
      "Clinton=Clearly:0.432931300479\n",
      "Clinton=Criticism:0.412549752042\n",
      "Clinton=Advisers:0.513010552981\n",
      "Clinton=WEEK:0.439776199289\n",
      "Clinton=Pressed:0.420844717541\n",
      "Clinton=PERSONALLY:0.403702946774\n",
      "Clinton=aides:0.669289576086\n",
      "Clinton=CLEAR:0.415697642445\n",
      "Clinton=VOTERS:0.461827240041\n",
      "Clinton=BOTH:0.414944988705\n",
      "Clinton=PUTIN:0.431702679126\n",
      "Clinton=advisers:0.513010552981\n",
      "Clinton=NOMINATION:0.442913878405\n",
      "Clinton=argued:0.402304347321\n",
      "Clinton=Rumsfeld:0.400114825567\n",
      "Clinton=PLEDGE:0.439916804207\n",
      "Clinton=Ago:0.405617239352\n",
      "Clinton=DEMOCRAT:0.431289937183\n",
      "Clinton=Though:0.426896891279\n",
      "Clinton=aide:0.515975902892\n",
      "Clinton=MCCAIN:0.644689815975\n",
      "Clinton=bush:0.788992254852\n",
      "Clinton=PRESIDENCY:0.443548479795\n",
      "Clinton=Candidacy:0.411260862179\n",
      "Clinton=ROMNEY:0.443267250573\n",
      "Clinton=KERRY:0.540085205891\n",
      "Clinton=EVEN:0.458548130487\n",
      "Clinton=AGO:0.405617239352\n",
      "Clinton=democratic:0.491501049884\n",
      "Clinton=ADMINISTRATION:0.601520551405\n",
      "Clinton=seemed:0.435614437499\n",
      "Clinton=REPUBLICANS:0.612489619751\n",
      "Clinton=Albright:0.527396976815\n",
      "Clinton=Gramm:0.40957529096\n",
      "Clinton=acknowledged:0.465075474932\n",
      "Clinton=hoped:0.436225098532\n",
      "Clinton=TUESDAY:0.427181202231\n",
      "Clinton=republicans:0.612489619751\n",
      "Clinton=Recently:0.403519250302\n",
      "Clinton=obama:0.70856035988\n",
      "Clinton=proposals:0.424398795279\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( \"Clinton\", 0.40 )\n",
    "for word in candidates:\n",
    "    print( word )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pence=Pence:1.0\n",
      "Pence=Rose:0.41822954375\n",
      "Pence=share:0.417077539644\n",
      "Pence=fell:0.40407342919\n",
      "Pence=cents:0.505148085458\n",
      "Pence=pence:1.0\n",
      "Pence=rose:0.41822954375\n",
      "Pence=PENCE:1.0\n",
      "Pence=SHARE:0.417077539644\n",
      "Pence=shares:0.483522537554\n",
      "Pence=Share:0.417077539644\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( \"Pence\", 0.40 )\n",
    "for word in candidates:\n",
    "    print( word )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Putin=Obama:0.419637820223\n",
      "Putin=President:0.473754361392\n",
      "Putin=Clinton:0.431702679126\n",
      "Putin=Russia:0.619720125241\n",
      "Putin=Bush:0.555893119254\n",
      "Putin=leaders:0.425382055435\n",
      "Putin=president:0.473754361392\n",
      "Putin=saying:0.46330910568\n",
      "Putin=visit:0.42971172742\n",
      "Putin=Prime:0.486428477051\n",
      "Putin=Minister:0.410939854436\n",
      "Putin=Putin:1.0\n",
      "Putin=Russian:0.601302176084\n",
      "Putin=Summit:0.435486820738\n",
      "Putin=talks:0.41399081474\n",
      "Putin=promised:0.415765893669\n",
      "Putin=Russians:0.439813324929\n",
      "Putin=Moscow:0.618918831565\n",
      "Putin=presidency:0.403113323776\n",
      "Putin=Presidency:0.403113323776\n",
      "Putin=Visit:0.42971172742\n",
      "Putin=Ukraine:0.473330073598\n",
      "Putin=Assad:0.422141295665\n",
      "Putin=Leaders:0.425382055435\n",
      "Putin=CLINTON:0.431702679126\n",
      "Putin=PRESIDENT:0.473754361392\n",
      "Putin=Talks:0.41399081474\n",
      "Putin=BUSH:0.555893119254\n",
      "Putin=Mubarak:0.44221879389\n",
      "Putin=RUSSIA:0.619720125241\n",
      "Putin=Vladimir:0.72623916325\n",
      "Putin=Merkel:0.518561680206\n",
      "Putin=prime:0.486428477051\n",
      "Putin=OBAMA:0.419637820223\n",
      "Putin=Karzai:0.456150698087\n",
      "Putin=Chavez:0.420222893279\n",
      "Putin=Blair:0.496299150926\n",
      "Putin=Medvedev:0.757306296389\n",
      "Putin=insisted:0.40122303176\n",
      "Putin=summit:0.435486820738\n",
      "Putin=Ahmadinejad:0.483973647393\n",
      "Putin=Ukrainian:0.405032332721\n",
      "Putin=Abbas:0.416745184281\n",
      "Putin=Saying:0.46330910568\n",
      "Putin=SAYING:0.46330910568\n",
      "Putin=Boris:0.531729928886\n",
      "Putin=LEADERS:0.425382055435\n",
      "Putin=MOSCOW:0.618918831565\n",
      "Putin=Soviet:0.412540988041\n",
      "Putin=Nursultan:0.404800762819\n",
      "Putin=Nazarbayev:0.517833913143\n",
      "Putin=Kazakhstan:0.401437596728\n",
      "Putin=Erdogan:0.468244302006\n",
      "Putin=Sergei:0.443755455934\n",
      "Putin=Lavrov:0.494595172134\n",
      "Putin=PUTIN:1.0\n",
      "Putin=minister:0.410939854436\n",
      "Putin=Rumsfeld:0.420819860594\n",
      "Putin=hu:0.523771391958\n",
      "Putin=bush:0.555893119254\n",
      "Putin=PRESIDENCY:0.403113323776\n",
      "Putin=SUMMIT:0.435486820738\n",
      "Putin=Gerhard:0.402094215286\n",
      "Putin=Chechnya:0.471946179945\n",
      "Putin=Saakashvili:0.548907582964\n",
      "Putin=obama:0.419637820223\n"
     ]
    }
   ],
   "source": [
    "candidates = get_similar_tokens( \"Putin\", 0.40 )\n",
    "for word in candidates:\n",
    "    print( word )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "def get_related( word ):\n",
    "    \n",
    "    filtered_words = [ w for w in word.vocab if w.is_lower == word.is_lower and w.prob >= -15 ]\n",
    "    #similarity = sorted( filtered_words, key=lambda w: word.similarity( w ), reverse=True )\n",
    "    #return similarity[ :10 ]\n",
    "    return filtered_words\n",
    "\n",
    "print( [ w.lower_ for w in get_related( nlp.vocab[ u'president' ] ) ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.lexeme.Lexeme at 0x7f31fa854900>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab[ u'president' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.624673\n",
      "0.183959\n",
      "0.180288\n"
     ]
    }
   ],
   "source": [
    "nlp2 = spacy.load( 'en_core_web_lg', parser=False )\n",
    "print( nlp2.vocab[u'dog'].similarity(nlp2.vocab[u'dachshund']) )\n",
    "print( nlp2.vocab[ u'trump' ].similarity( nlp2.vocab[ u'president' ] ) )\n",
    "print( nlp2.vocab[ u'trump' ].similarity( nlp2.vocab[ u'man' ] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(7732777389095836264, 0, 2)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from spacy.matcher import PhraseMatcher\n",
    "\n",
    "matcher = PhraseMatcher( nlp.vocab )\n",
    "matcher.add( 'OBAMA', None, nlp( u\"Barack Obama\" ) )\n",
    "doc = nlp( u\"Barack Obama lifts America one last time in emotional farewell\" )\n",
    "matches = matcher(doc)\n",
    "\n",
    "matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog dog 1.0\n",
      "dog cat 0.53907\n",
      "dog banana 0.28761\n",
      "cat dog 0.53907\n",
      "cat cat 1.0\n",
      "cat banana 0.487522\n",
      "banana dog 0.28761\n",
      "banana cat 0.487522\n",
      "banana banana 1.0\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load( 'en' )  # make sure to use larger model!\n",
    "tokens = nlp( u'dog cat banana' )\n",
    "\n",
    "for token1 in tokens:\n",
    "    for token2 in tokens:\n",
    "        print(token1.text, token2.text, token1.similarity(token2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog dog 1.0\n",
      "dog cat 0.801686\n",
      "dog banana 0.243276\n",
      "cat dog 0.801686\n",
      "cat cat 1.0\n",
      "cat banana 0.281544\n",
      "banana dog 0.243276\n",
      "banana cat 0.281544\n",
      "banana banana 1.0\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load( 'en_core_web_lg' )  # make sure to use larger model!\n",
    "tokens = nlp( u'dog cat banana' )\n",
    "\n",
    "for token1 in tokens:\n",
    "    for token2 in tokens:\n",
    "        print( token1.text, token2.text, token1.similarity( token2 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obama Obama 1.0\n",
      "Obama Clinton 0.782694\n",
      "Obama Criminal 0.246474\n",
      "Obama Trump 0.42475\n",
      "Clinton Obama 0.782694\n",
      "Clinton Clinton 1.0\n",
      "Clinton Criminal 0.219609\n",
      "Clinton Trump 0.415443\n",
      "Criminal Obama 0.246474\n",
      "Criminal Clinton 0.219609\n",
      "Criminal Criminal 1.0\n",
      "Criminal Trump 0.217229\n",
      "Trump Obama 0.42475\n",
      "Trump Clinton 0.415443\n",
      "Trump Criminal 0.217229\n",
      "Trump Trump 1.0\n"
     ]
    }
   ],
   "source": [
    "tokens = nlp( u'Obama Clinton Criminal Trump' )\n",
    "\n",
    "for token1 in tokens:\n",
    "    for token2 in tokens:\n",
    "        print( token1.text, token2.text, token1.similarity( token2 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Apple apple PROPN NNP nsubj Xxxxx True False\n",
      "is be VERB VBZ aux xx True False\n",
      "looking look VERB VBG ROOT xxxx True False\n",
      "at at ADP IN prep xx True False\n",
      "buying buy VERB VBG pcomp xxxx True False\n",
      "U.K. u.k. PROPN NNP compound X.X. False False\n",
      "startup startup NOUN NN dobj xxxx True False\n",
      "for for ADP IN prep xxx True False\n",
      "$ $ SYM $ quantmod $ False False\n",
      "1 1 NUM CD compound d False False\n",
      "billion billion NUM CD pobj xxxx True False\n"
     ]
    }
   ],
   "source": [
    "doc = nlp( u'Apple is looking at buying U.K. startup for $1 billion' )\n",
    "#doc = nlp( u\"In any event, Washington is broken, and our country is in serious trouble and total disarray.  Very simple.  Politicians are all talk, no action.  They are all talk and no action.  And it's constant; it never ends.\" )\n",
    "\n",
    "for token in doc:\n",
    "    print( token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Apple 0 5 ORG\n",
      "U.K. 27 31 GPE\n",
      "$1 billion 44 54 MONEY\n"
     ]
    }
   ],
   "source": [
    "doc = nlp( u'Apple is looking at buying U.K. startup for $1 billion' )\n",
    "\n",
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.start_char, ent.end_char, ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[93m    Serving on port 5000...\u001b[0m\n",
      "    Using the 'dep' visualizer\n",
      "\n",
      "\n",
      "    Shutting down server on port 5000.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# THIS HANGS. PROBABLY NEEDS PORT 5000 tunneled... NOPE!\n",
    "from spacy import displacy\n",
    "\n",
    "doc = nlp(u'This is a sentence.')\n",
    "displacy.serve(doc, style='dep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', 'world', '.', 'Here', 'are', 'two', 'sentences', '.']\n"
     ]
    }
   ],
   "source": [
    "doc = nlp( u'Hello, world. Here are two sentences.' )\n",
    "print([t.text for t in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Peach\n",
      "emoji\n",
      "🍑\n",
      "outranking eggplant\n",
      "start?\n",
      "Peach emoji\n",
      "it\n",
      "Peach\n",
      "the superior emoji\n",
      "It\n",
      "eggplant 🍑\n",
      "end?\n",
      "Peach is the superior emoji.\n"
     ]
    }
   ],
   "source": [
    "doc = nlp( u\"Peach emoji is where it has always been. Peach is the superior emoji. It's outranking eggplant 🍑 \")\n",
    "print(doc[0].text)          # Peach\n",
    "print(doc[1].text)          # emoji\n",
    "print(doc[-1].text)         # 🍑\n",
    "print(doc[17:19].text)      # outranking eggplant\n",
    "\n",
    "noun_chunks = list( doc.noun_chunks )\n",
    "print( \"start?\" )\n",
    "for chunk in noun_chunks:\n",
    "    print( chunk.text )  # Peach emoji\n",
    "print( \"end?\" )\n",
    "sentences = list(doc.sents )\n",
    "assert len(sentences) == 3\n",
    "print(sentences[1].text)    # 'Peach is the superior emoji.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-grained POS tag PROPN 95\n",
      "Coarse-grained POS tag NNP 15794550382381185553\n",
      "Word shape Xxxxx 16072095006890171862\n",
      "Alphanumeric characters? True\n",
      "Punctuation mark? False\n",
      "Digit? False\n",
      "Like a number? True\n",
      "Like an email address? False\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(u'Apple is looking at buying U.K. startup for $1 billion')\n",
    "apple = doc[0]\n",
    "print('Fine-grained POS tag', apple.pos_, apple.pos )\n",
    "print('Coarse-grained POS tag', apple.tag_, apple.tag )\n",
    "print('Word shape', apple.shape_, apple.shape )\n",
    "print('Alphanumeric characters?', apple.is_alpha )\n",
    "print('Punctuation mark?', apple.is_punct )\n",
    "\n",
    "billion = doc[10]\n",
    "print('Digit?', billion.is_digit)\n",
    "print('Like a number?', billion.like_num)\n",
    "print('Like an email address?', billion.like_email)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "San Francisco 0 13 GPE\n",
      "FB 0 2 ORG\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(u'San Francisco considers banning sidewalk delivery robots')\n",
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.start_char, ent.end_char, ent.label_)\n",
    "\n",
    "from spacy.tokens import Span\n",
    "doc = nlp(u'FB is hiring a new VP of global policy')\n",
    "doc.ents = [Span(doc, 0, 1, label=doc.vocab.strings[u'ORG'])]\n",
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.start_char, ent.end_char, ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[93m    Serving on port 5000...\u001b[0m\n",
      "    Using the 'dep' visualizer\n",
      "\n",
      "\n",
      "    Shutting down server on port 5000.\n",
      "\n",
      "\n",
      "\u001b[93m    Serving on port 5000...\u001b[0m\n",
      "    Using the 'ent' visualizer\n",
      "\n",
      "\n",
      "    Shutting down server on port 5000.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from spacy import displacy\n",
    "\n",
    "doc_dep = nlp(u'This is a sentence.')\n",
    "displacy.serve(doc_dep, style='dep')\n",
    "\n",
    "doc_ent = nlp(u'When Sebastian Thrun started working on self-driving cars at Google '\n",
    "              u'in 2007, few people outside of the company took him seriously.')\n",
    "displacy.serve(doc_ent, style='ent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apple <-> banana 0.583184\n",
      "pasta <-> hippo 0.0793491\n",
      "True True True True\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(u\"Apple and banana are similar. Pasta and hippo aren't.\")\n",
    "\n",
    "apple = doc[0]\n",
    "banana = doc[2]\n",
    "pasta = doc[6]\n",
    "hippo = doc[8]\n",
    "\n",
    "print('apple <-> banana', apple.similarity(banana))\n",
    "print('pasta <-> hippo', pasta.similarity(hippo))\n",
    "print(apple.has_vector, banana.has_vector, pasta.has_vector, hippo.has_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['advmod', 'advcl', 'compound', 'nsubj', 'advcl', 'nsubj', 'advcl', 'advcl', 'xcomp', 'advcl', 'prep', 'xcomp', 'advcl', 'npadvmod', 'amod', 'pobj', 'prep', 'xcomp', 'advcl', 'punct', 'amod', 'pobj', 'prep', 'xcomp', 'advcl', 'amod', 'pobj', 'prep', 'xcomp', 'advcl', 'pobj', 'prep', 'xcomp', 'advcl', 'prep', 'xcomp', 'advcl', 'pobj', 'prep', 'xcomp', 'advcl', 'prep', 'xcomp', 'advcl', 'pobj', 'prep', 'xcomp', 'advcl', 'punct', 'amod', 'nsubj', 'nsubj', 'advmod', 'nsubj', 'prep', 'advmod', 'nsubj', 'det', 'pobj', 'prep', 'advmod', 'nsubj', 'pobj', 'prep', 'advmod', 'nsubj', 'dobj', 'advmod', 'punct']\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(u\"When Sebastian Thrun started working on self-driving cars at Google \"\n",
    "          u\"in 2007, few people outside of the company took him seriously.\")\n",
    "\n",
    "dep_labels = []\n",
    "for token in doc:\n",
    "    while token.head != token:\n",
    "        dep_labels.append(token.dep_)\n",
    "        token = token.head\n",
    "print(dep_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<pre><span class=\"pos-DET dep-nsubj\">This</span> <span class=\"pos-VERB dep-ROOT\">is</span> <span class=\"pos-DET dep-det\">a</span> <span class=\"pos-NOUN dep-attr\">test</span><span class=\"pos-PUNCT dep-punct\">.</span><span class=\"pos-INTJ dep-nmod\">Hello</span>   <span class=\"pos-NOUN dep-ROOT\">world</span><span class=\"pos-PUNCT dep-punct\">.</span></pre>'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def put_spans_around_tokens(doc):\n",
    "    \"\"\"Here, we're building a custom \"syntax highlighter\" for\n",
    "    part-of-speech tags and dependencies. We put each token in a\n",
    "    span element, with the appropriate classes computed. All whitespace is\n",
    "    preserved, outside of the spans. (Of course, HTML will only display\n",
    "    multiple whitespace if enabled – but the point is, no information is lost\n",
    "    and you can calculate what you need, e.g. <br />, <p> etc.)\n",
    "    \"\"\"\n",
    "    output = []\n",
    "    html = '<span class=\"{classes}\">{word}</span>{space}'\n",
    "    for token in doc:\n",
    "        if token.is_space:\n",
    "            output.append(token.text)\n",
    "        else:\n",
    "            classes = 'pos-{} dep-{}'.format(token.pos_, token.dep_)\n",
    "            output.append(html.format(classes=classes, word=token.text, space=token.whitespace_))\n",
    "    string = ''.join(output)\n",
    "    string = string.replace('\\n', '')\n",
    "    string = string.replace('\\t', '    ')\n",
    "    return '<pre>{}</pre>'.format(string)\n",
    "\n",
    "doc = nlp(u\"This is a test.\\n\\nHello   world.\")\n",
    "html = put_spans_around_tokens(doc)\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "html\n",
    "#display( HTML( html ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Tweet Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22322"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}