6_Word_embeddings_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Thanks to Jeff Delaney kaggle.com/jeffd23 for this simple tutorial on Kaggle https://www.kaggle.com/jeffd23/visualizing-word-vectors-with-t-sne\n",
    "\n",
    "#Visualizing Word Vectors with t-SNE\n",
    "# TSNE is pretty useful when it comes to visualizing similarity between objects. It works by taking a group of high-dimensional (100 dimensions via Word2Vec) vocabulary word feature vectors, then compresses them down to 2-dimensional x,y coordinate pairs. The idea is to keep similar words close together on the plane, while maximizing the distance between dissimilar words.\n",
    "\n",
    "#Steps\n",
    "#1.Clean the data\n",
    "#2.Build a corpus\n",
    "#3.Train a Word2Vec Model\n",
    "#4.Visualize t-SNE representations of the most common words\n",
    "\n",
    "\n",
    "#TIP: On jupyter you can use %%capture first thing in a cell to catch warnings (warning this stops all sort of output)\n",
    "#Otherwise use something like:\n",
    "#with warnings.catch_warnings():\n",
    "#    warnings.simplefilter('ignore')\n",
    "#    # Your problematic instruction(s) here'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "pd.options.mode.chained_assignment = None \n",
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "\n",
    "from gensim.models import word2vec\n",
    "\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "combinedFile = \"csvs/combined_csv.csv\"\n",
    "\n",
    "df = pd.read_csv(combinedFile, delimiter=',', header=0)\n",
    "\n",
    "\n",
    "print('Number of titles: {:,}\\n'.format(df.shape[0]))\n",
    "df.sample(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove chars that are not letters or numbers, downcase, then remove stop words\n",
    "\n",
    "import nltk\n",
    "import re\n",
    "import math\n",
    "#nltk.download('stopwords') #uncomment if not downloaded\n",
    "en_stop = set(nltk.corpus.stopwords.words('english'))\n",
    "de_stop = set(nltk.corpus.stopwords.words('german'))\n",
    "\n",
    "def clean_sentence(text):\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    text = text.replace('>', ' ').replace('<', ' ')\n",
    "    text = re.sub(' +', ' ', text)\n",
    "    text = text.replace('hackz', 'hack').replace('securityz', 'security').replace('attackz','attacks').replace('Anfngerz','')\n",
    "    regex = re.compile('([^\\s\\w]|_)+')\n",
    "    sentence = regex.sub('', text).lower()\n",
    "    sentence = sentence.split(\" \")\n",
    "    \n",
    "    for word in list(sentence):\n",
    "        if word in en_stop or word in de_stop or len(word) < 3:\n",
    "            sentence.remove(word)  \n",
    "            \n",
    "    sentence = \" \".join(sentence)\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates a list of lists containing words from each sentence\n",
    "def build_corpus(columnData):\n",
    "    corpus = []\n",
    "    for i in range(len(df)):\n",
    "        if str(columnData[i]) != 'nan':\n",
    "            sentence = clean_sentence(columnData[i])\n",
    "            word_list = sentence.split(\" \")\n",
    "            corpus.append(word_list)\n",
    "    return corpus\n",
    "\n",
    "corpus = build_corpus(df[\"title\"])\n",
    "# classify that the article has recommends\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"[\",df[\"title\"][0],\"],[\",df[\"title\"][1],\"]\")#original first two sentences as in the CSV file\n",
    "print(corpus[0:2])#first two sentences after cleaning and removing stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Word2Vec model\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.models import Word2Vec\n",
    "#The Word to Vec model produces a vocabulary, with each word being represented by an n-dimensional numpy array (100 values in this example)\n",
    "model = Word2Vec(sentences=corpus, size=100, window=5, min_count=1, workers=4,sg=0)#sg refers to skip-gram and here it's off and by default we are using a CBOW. #CBOW tries to predict a word on the basis of its neighbors, while Skip Gram tries to predict the neighbors of a word. In simpler words, CBOW tends to find the probability of a word occurring in a context. So, it generalizes over all the different contexts in which a word can be used.\n",
    "model.save(\"models/word2vec.model\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv['computer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#OOV!! In some cases and especially with small training data, some words are not observed when train- ing the embedding, which are known as out-of- vocabulary (OOV) words.\n",
    "try:\n",
    "    oov_word = \"Lancaster\"\n",
    "    model.wv[oov_word]\n",
    "except KeyError:\n",
    "    print(\"Oops! The word\",\"[\",oov_word,\"] not in vocabulary!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#Plotting similarities of a word embedding model using a scatter plot from t-SNE\n",
    "#(t-SNE) t-Distributed Stochastic Neighbor Embedding is a non-linear dimensionality reduction algorithm used for exploring high-dimensional data. It maps multi-dimensional data to two or more dimensions suitable for human observation. With help of the t-SNE algorithms, you may have to plot fewer exploratory data analysis plots next time you work with high dimensional data.\n",
    "def tsne_plot(model,modelName):\n",
    "    \"Creates and TSNE model and plots it\"\n",
    "    labels = []\n",
    "    tokens = []\n",
    "\n",
    "    for word in model.wv.vocab:\n",
    "        tokens.append(model[word])\n",
    "        labels.append(word)\n",
    "    \n",
    "    tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
    "    new_values = tsne_model.fit_transform(tokens)\n",
    "\n",
    "    x = []\n",
    "    y = []\n",
    "    for value in new_values:\n",
    "        x.append(value[0])\n",
    "        y.append(value[1])\n",
    "        \n",
    "    plt.figure(figsize=(16, 16)) \n",
    "    for i in range(len(x)):\n",
    "        plt.scatter(x[i],y[i])\n",
    "        plt.annotate(labels[i],\n",
    "                     xy=(x[i], y[i]),\n",
    "                     xytext=(5, 2),\n",
    "                     textcoords='offset points',\n",
    "                     ha='right',\n",
    "                     va='bottom')\n",
    "    pltFileName = 'plots'+'/'+'word_embeddings'+'_'+modelName+'.pdf';\n",
    "    plt.savefig(pltFileName)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model10 = Word2Vec(sentences=corpus, size=100, window=5, min_count=10, workers=4,sg=0)#sg refers to skip-gram and here it's off and by default we are using a CBOW. #CBOW tries to predict a word on the basis of its neighbors, while Skip Gram tries to predict the neighbors of a word. In simpler words, CBOW tends to find the probability of a word occurring in a context. So, it generalizes over all the different contexts in which a word can be used.\n",
    "model10.save(\"models/word2vec10.model\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter('ignore')\n",
    "    #plotting the w2v model for words occuring more than 50 times\n",
    "    tsne_plot(model10,\"model10\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "import seaborn as sns\n",
    "sns.set_style(\"darkgrid\")\n",
    "\n",
    "\n",
    "def tsnescatterplot(model, word, list_names):\n",
    "    \"\"\" Plot in seaborn the results from the t-SNE dimensionality reduction algorithm of the vectors of a query word,\n",
    "    its list of most similar words, and a list of words.\n",
    "    \"\"\"\n",
    "    arrays = np.empty((0, 100), dtype='f')\n",
    "    word_labels = [word]\n",
    "    color_list  = ['red']\n",
    "\n",
    "    # adds the vector of the query word\n",
    "    arrays = np.append(arrays, model.wv.__getitem__([word]), axis=0)\n",
    "    \n",
    "    # gets list of most similar words\n",
    "    close_words = model.wv.most_similar([word])\n",
    "    \n",
    "    # adds the vector for each of the closest words to the array\n",
    "    for wrd_score in close_words:\n",
    "        wrd_vector = model.wv.__getitem__([wrd_score[0]])\n",
    "        word_labels.append(wrd_score[0])\n",
    "        color_list.append('blue')\n",
    "        arrays = np.append(arrays, wrd_vector, axis=0)\n",
    "    \n",
    "\n",
    "        \n",
    "    # Reduces the dimensionality from 100 to 11 dimensions with PCA\n",
    "    reduc = PCA(n_components=10, ).fit_transform(arrays)\n",
    "    \n",
    "    # Finds t-SNE coordinates for 2 dimensions\n",
    "    np.set_printoptions(suppress=True)\n",
    "    \n",
    "    Y = TSNE(n_components=2, random_state=0, perplexity=15).fit_transform(reduc)\n",
    "    \n",
    "    # Sets everything up to plot\n",
    "    df = pd.DataFrame({'x': [x for x in Y[:, 0]],\n",
    "                       'y': [y for y in Y[:, 1]],\n",
    "                       'words': word_labels,\n",
    "                       'color': color_list})\n",
    "    \n",
    "    fig, _ = plt.subplots()\n",
    "    fig.set_size_inches(9, 9)\n",
    "    \n",
    "    # Basic plot\n",
    "    p1 = sns.regplot(data=df,\n",
    "                     x=\"x\",\n",
    "                     y=\"y\",\n",
    "                     fit_reg=False,\n",
    "                     marker=\"o\",\n",
    "                     scatter_kws={'s': 40,\n",
    "                                  'facecolors': df['color']\n",
    "                                 }\n",
    "                    )\n",
    "    \n",
    "    # Adds annotations one by one with a loop\n",
    "    for line in range(0, df.shape[0]):\n",
    "         p1.text(df[\"x\"][line],\n",
    "                 df['y'][line],\n",
    "                 '  ' + df[\"words\"][line].title(),\n",
    "                 horizontalalignment='left',\n",
    "                 verticalalignment='bottom', size='medium',\n",
    "                 color=df['color'][line],\n",
    "                 weight='normal'\n",
    "                ).set_size(15)\n",
    "\n",
    "    \n",
    "    plt.xlim(Y[:, 0].min()-50, Y[:, 0].max()+50)\n",
    "    plt.ylim(Y[:, 1].min()-50, Y[:, 1].max()+50)\n",
    "            \n",
    "    plt.title('t-SNE visualization for {}'.format(word.title()))\n",
    "    pltFileName = 'plots'+'/'+'word_embeddings_similarity_to'+'_'+word+'.pdf';\n",
    "    plt.savefig(pltFileName)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_word = 'security'\n",
    "model.most_similar(wv_word)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsnescatterplot(model, wv_word, [i[0] for i in model.wv.most_similar(wv_word)]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}