Data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from collections import defaultdict\n",
    "import os\n",
    "import sys\n",
    "import re\n",
    "import numpy as np\n",
    "import preprocessor as p\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "import operator\n",
    "import pickle\n",
    "\n",
    "p.set_options(p.OPT.URL, p.OPT.NUMBER, p.OPT.RESERVED, p.OPT.MENTION)\n",
    "\n",
    "TWEET_EMBEDDING_PATH = \".glove.twitter.27B.200d.txt\"\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 40\n",
    "MAX_NUM_WORDS = 40000\n",
    "EMBEDDING_DIM = 200\n",
    "\n",
    "\n",
    "def load_sentences_from_df(train_df, id_field = 'id', sentiment_label='label', tweet_field='tweet', lower=True, clean=True):\n",
    "    \"\"\"\n",
    "    Loads sentences.\n",
    "    :param train_df: pandas.DataFrame containing labeled tweets.\n",
    "    :return: sents (paired with labels), word doc freq, list of labels.\n",
    "    \"\"\"\n",
    "    sents = []\n",
    "    lbl = {'negative':0,\n",
    "           'neutral':1,\n",
    "          'positive':2}\n",
    "    ids = set()\n",
    "    word_df = defaultdict(int)        \n",
    "    for line in train_df.iterrows():\n",
    "        \n",
    "        if not(line[1][id_field] in ids):\n",
    "            ids.add(line[1][id_field])\n",
    "            tweet = line[1][tweet_field]\n",
    "            sentiment = line[1][sentiment_label]\n",
    "\n",
    "            clean_text = tweet.lower() if lower else text\n",
    "            clean_text = p.clean(clean_text) if clean else clean_text\n",
    "            clean_text = re.sub(r'(\\\\u[0-9A-Fa-f]+)',r'', re.sub(r'[^\\x00-\\x7f]',r'', clean_text)) if clean else clean_text\n",
    "\n",
    "            words = clean_text.split()\n",
    "            for word in set(words):\n",
    "                word_df[word] += 1\n",
    "            pair = (words, lbl[sentiment])\n",
    "            sents.append(pair)\n",
    "\n",
    "    labels = [0] * len(lbl)\n",
    "    for l,i in lbl.items():\n",
    "        labels[i] = l\n",
    "        \n",
    "    return sents, word_df, labels\n",
    "\n",
    "def split(df, pct):\n",
    "    start = 0\n",
    "    end = 0\n",
    "    result = []\n",
    "\n",
    "    for i in range(0,len(pct)):\n",
    "        end = start+int(len(df)*pct[i])\n",
    "        result.append(df.iloc[start:end])\n",
    "        start=end\n",
    "    \n",
    "    return result  \n",
    "\n",
    "def df_from_tsv(path):\n",
    "    data = []\n",
    "    with open(path, \"r\") as f:\n",
    "        for l in f:\n",
    "            rec = l[:-1].split('\\t')\n",
    "            if len(rec) == 3:\n",
    "                data.append(rec)\n",
    "    return pd.DataFrame(columns=['id', 'label', 'tweet'], data=data)\n",
    "\n",
    "def createDatasets(df, labels, column, pct, shuffle=False):\n",
    "    \n",
    "    results_tmp = []\n",
    "    result = []\n",
    "    i=0\n",
    "    \n",
    "    for l in labels:\n",
    "        d_tmp = df[df[column]==l]\n",
    "        results_tmp.append(split(d_tmp,pct))\n",
    "\n",
    "\n",
    "    for i in range(0,len(labels)):\n",
    "        d=pd.DataFrame()\n",
    "        for j in range(0,len(labels)):\n",
    "            d=d.append(results_tmp[j][i])\n",
    "        if shuffle:\n",
    "            d=d.reindex(np.random.RandomState(seed=2).permutation(d.index))\n",
    "        result.append(d)\n",
    "\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train with dup has 21826 records\n",
      "Train with no dup has 21240 records, 586 less.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000794790727680</td>\n",
       "      <td>positive</td>\n",
       "      <td>One Night like In Vegas I make dat Nigga Famous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100000831528632320</td>\n",
       "      <td>positive</td>\n",
       "      <td>Walking through Chelsea at this time of day is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100000950005145600</td>\n",
       "      <td>neutral</td>\n",
       "      <td>\"And on the very first play of the night, Aaro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100000974885748736</td>\n",
       "      <td>neutral</td>\n",
       "      <td>\"Drove the bike today, about 40 miles. Felt li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100001038454624257</td>\n",
       "      <td>negative</td>\n",
       "      <td>looking at the temp outside....hpw did it get ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id     label  \\\n",
       "0  100000794790727680  positive   \n",
       "1  100000831528632320  positive   \n",
       "2  100000950005145600   neutral   \n",
       "3  100000974885748736   neutral   \n",
       "4  100001038454624257  negative   \n",
       "\n",
       "                                               tweet  \n",
       "0    One Night like In Vegas I make dat Nigga Famous  \n",
       "1  Walking through Chelsea at this time of day is...  \n",
       "2  \"And on the very first play of the night, Aaro...  \n",
       "3  \"Drove the bike today, about 40 miles. Felt li...  \n",
       "4  looking at the temp outside....hpw did it get ...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dfs = [ df_from_tsv(\"./data/train/twitter-train\"+str(i)+\".txt\") for i in range(6)]\n",
    "train_df = pd.concat(train_dfs)\n",
    "print(\"Train with dup has \" + str(len(train_df)) + \" records\")\n",
    "\n",
    "#removing duplicates\n",
    "no_dup = train_df.groupby(as_index=False, by=['id']).first()\n",
    "print(\"Train with no dup has \" + str(len(no_dup)) + \" records, \"+str(len(train_df)-len(no_dup))+\" less.\")\n",
    "no_dup.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------- Train -------\n",
      "number of sentences: 21240\n",
      "vocab size: 48625\n",
      "max sentence length: 33\n"
     ]
    }
   ],
   "source": [
    "#It's possible to divide the dataset in three part: test, train and validation. We decided to keep the dateset intact\n",
    "tweet_df=createDatasets(no_dup,['positive','negative','neutral'],'label',[0.,1.,0.],shuffle=True)\n",
    "train=tweet_df[1]\n",
    "\n",
    "#statistics\n",
    "train_sents, word_df, train_labels = load_sentences_from_df(train)\n",
    "max_l = max(len(words) for words,l in train_sents)\n",
    "print('------- Train -------')\n",
    "print( \"number of sentences: %d\" % len(train_sents))\n",
    "print( \"vocab size: %d\" % len(word_df))\n",
    "print( \"max sentence length: %d\" % max_l)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build embedding index from GloVe-Twitter-.27B-200d "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we build the embeddings composed of the only words present in our tweets\n",
    "embeddings_index = {}\n",
    "with open(TWEET_EMBEDDING_PATH, mode='rb') as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        if(word_df.get(word) is not None): #if the word is present in word_df, we keep its values\n",
    "            coefs = np.asarray(values[1:], dtype='float32')\n",
    "            embeddings_index[word] = coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 21240 tweets.\n"
     ]
    }
   ],
   "source": [
    "text = []\n",
    "labls = []\n",
    "\n",
    "for s in train_sents:\n",
    "    text.append(s[0])\n",
    "    labls.append(s[1])\n",
    "\n",
    "print('Found %s tweets.' % len(train_sents))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 42130 unique tokens.\n",
      "Shape of data tensor: (16991, 40)\n",
      "Shape of label tensor: (16991, 3)\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, lower=True)\n",
    "tokenizer.fit_on_texts(text)\n",
    "sequences = tokenizer.texts_to_sequences(text)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "train_data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "train_labels = to_categorical(np.asarray(labls))\n",
    "print('Shape of data tensor:', train_data.shape)\n",
    "print('Shape of label tensor:', train_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build embedding matrix for the lookup layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words = min(MAX_NUM_WORDS, len(word_index))\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "high = 2.38 / np.sqrt(len(text) + EMBEDDING_DIM) # see (Bottou '88)\n",
    "for word, i in word_index.items():\n",
    "    if i >= num_words:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "    else: \n",
    "        embedding_matrix[i] = np.random.uniform(-high, high, EMBEDDING_DIM)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(file = open('embedding_matrixG200', 'wb'), obj = embedding_matrix)\n",
    "pickle.dump(file = open('train_data', 'wb'), obj = train_data)\n",
    "pickle.dump(file = open('train_labels', 'wb'), obj = train_labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}