emojify.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Input, Dropout, LSTM, Activation\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "from keras.initializers import glorot_uniform\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sentences_to_indices(X, word_to_index, max_len):\n",
    "    \"\"\"\n",
    "    Converts an array of sentences (strings) into an array of indices corresponding to words in the sentences.\n",
    "    The output shape should be such that it can be given to `Embedding()` (described in Figure 4). \n",
    "    \n",
    "    Arguments:\n",
    "    X -- array of sentences (strings), of shape (m, 1)\n",
    "    word_to_index -- a dictionary containing the each word mapped to its index\n",
    "    max_len -- maximum number of words in a sentence. You can assume every sentence in X is no longer than this. \n",
    "    \n",
    "    Returns:\n",
    "    X_indices -- array of indices corresponding to words in the sentences from X, of shape (m, max_len)\n",
    "    \"\"\"\n",
    "    \n",
    "    m = X.shape[0]                                   # number of training examples\n",
    "    \n",
    "    # Initialize X_indices as a numpy matrix of zeros and the correct shape (≈ 1 line)\n",
    "    X_indices = np.zeros((m,max_len))\n",
    "    \n",
    "    for i in range(m):                               # loop over training examples\n",
    "        \n",
    "        # Convert the ith training sentence in lower case and split is into words. You should get a list of words.\n",
    "        sentence_words = X[i].lower().split()\n",
    "        \n",
    "        # Initialize j to 0\n",
    "        j = 0\n",
    "        \n",
    "        # Loop over the words of sentence_words\n",
    "        for w in sentence_words:\n",
    "            # Set the (i,j)th entry of X_indices to the index of the correct word.\n",
    "            X_indices[i, j] = word_to_index[w]\n",
    "            # Increment j to j + 1\n",
    "            j = j+1\n",
    "    \n",
    "    return X_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X1 = ['funny lol' 'lets play baseball' 'food is ready for you']\n",
      "X1_indices = [[ 155345.  225122.       0.       0.       0.]\n",
      " [ 220930.  286375.   69714.       0.       0.]\n",
      " [ 151204.  192973.  302254.  151349.  394475.]]\n"
     ]
    }
   ],
   "source": [
    "X1 = np.array([\"funny lol\", \"lets play baseball\", \"food is ready for you\"])\n",
    "X1_indices = sentences_to_indices(X1,word_to_index, max_len = 5)\n",
    "\n",
    "#print (X1[1])\n",
    "print(\"X1 =\", X1)\n",
    "print(\"X1_indices =\", X1_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pretrained_embedding_layer(word_to_vec_map, word_to_index):\n",
    "    \"\"\"\n",
    "    Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.\n",
    "    \n",
    "    Arguments:\n",
    "    word_to_vec_map -- dictionary mapping words to their GloVe vector representation.\n",
    "    word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)\n",
    "\n",
    "    Returns:\n",
    "    embedding_layer -- pretrained layer Keras instance\n",
    "    \"\"\"\n",
    "    \n",
    "    vocab_len = len(word_to_index) + 1                  # adding 1 to fit Keras embedding (requirement)\n",
    "    emb_dim = word_to_vec_map[\"cucumber\"].shape[0]      # define dimensionality of your GloVe word vectors (= 50)\n",
    "    \n",
    "    # Initialize the embedding matrix as a numpy array of zeros of shape (vocab_len, dimensions of word vectors = emb_dim)\n",
    "    emb_matrix = np.zeros((vocab_len, emb_dim))\n",
    "    \n",
    "    # Set each row \"index\" of the embedding matrix to be the word vector representation of the \"index\"th word of the vocabulary\n",
    "    for word, index in word_to_index.items():\n",
    "        emb_matrix[index, :] = word_to_vec_map[word]\n",
    "\n",
    "    # Define Keras embedding layer with the correct output/input sizes, make it non-trainable. Use Embedding(...). Make sure to set trainable=False. \n",
    "    embedding_layer = Embedding(vocab_len, emb_dim, trainable = False) \n",
    "                      #trainable = False, gives rise to 20,000,050 (the word embeddings) as non-trainable\n",
    "\n",
    "    # Build the embedding layer, it is required before setting the weights of the embedding layer. Do not modify the \"None\".\n",
    "    embedding_layer.build((None,))\n",
    "    \n",
    "    # Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.\n",
    "    embedding_layer.set_weights([emb_matrix])\n",
    "    \n",
    "    return embedding_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights[0][1][3] = -0.3403\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
    "print(\"weights[0][1][3] =\", embedding_layer.get_weights()[0][1][3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Emojify_V2(input_shape, word_to_vec_map, word_to_index):\n",
    "    \"\"\"\n",
    "    Function creating the Emojify-v2 model's graph.\n",
    "    \n",
    "    Arguments:\n",
    "    input_shape -- shape of the input, usually (max_len,)\n",
    "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 50-dimensional vector representation\n",
    "    word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)\n",
    "\n",
    "    Returns:\n",
    "    model -- a model instance in Keras\n",
    "    \"\"\"\n",
    "\n",
    "    # Define sentence_indices as the input of the graph, it should be of shape input_shape and dtype 'int32' (as it contains indices).\n",
    "    sentence_indices = Input(shape = input_shape, dtype = \"int32\")\n",
    "    \n",
    "    # Create the embedding layer pretrained with GloVe Vectors (≈1 line)\n",
    "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
    "    \n",
    "    # Propagate sentence_indices through your embedding layer, you get back the embeddings\n",
    "    embeddings = embedding_layer(sentence_indices)\n",
    "    \n",
    "    # Propagate the embeddings through an LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a batch of sequences.\n",
    "    X = LSTM(units = 128, return_sequences = True)(embeddings)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(0.5)(X)\n",
    "    # Propagate X trough another LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a single hidden state, not a batch of sequences.\n",
    "    X = LSTM(units = 128, return_sequences = False)(X)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(0.5)(X)\n",
    "    # Propagate X through a Dense layer with softmax activation to get back a batch of 5-dimensional vectors.\n",
    "    X = Dense(5)(X)\n",
    "    # Add a softmax activation\n",
    "    X = Activation('softmax')(X)\n",
    "    \n",
    "    # Create Model instance which converts sentence_indices into X.\n",
    "    model = Model(inputs = sentence_indices, outputs = X)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "embedding_3 (Embedding)      (None, 10, 50)            20000050  \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 10, 128)           91648     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 10, 128)           0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 128)               131584    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 5)                 645       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 5)                 0         \n",
      "=================================================================\n",
      "Total params: 20,223,927\n",
      "Trainable params: 223,877\n",
      "Non-trainable params: 20,000,050\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Emojify_V2((maxLen,), word_to_vec_map, word_to_index)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_indices = sentences_to_indices(X_train, word_to_index, maxLen)\n",
    "Y_train_oh = convert_to_one_hot(Y_train, C = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "132/132 [==============================] - 0s - loss: 1.5929 - acc: 0.2273     \n",
      "Epoch 2/50\n",
      "132/132 [==============================] - 0s - loss: 1.5229 - acc: 0.3182     \n",
      "Epoch 3/50\n",
      "132/132 [==============================] - 0s - loss: 1.4740 - acc: 0.3409     \n",
      "Epoch 4/50\n",
      "132/132 [==============================] - 0s - loss: 1.4466 - acc: 0.4545     \n",
      "Epoch 5/50\n",
      "132/132 [==============================] - 0s - loss: 1.3738 - acc: 0.4621     \n",
      "Epoch 6/50\n",
      "132/132 [==============================] - 0s - loss: 1.2511 - acc: 0.5455     \n",
      "Epoch 7/50\n",
      "132/132 [==============================] - 0s - loss: 1.1269 - acc: 0.6515     \n",
      "Epoch 8/50\n",
      "132/132 [==============================] - 0s - loss: 1.0376 - acc: 0.5909     \n",
      "Epoch 9/50\n",
      "132/132 [==============================] - 0s - loss: 0.9226 - acc: 0.6439     \n",
      "Epoch 10/50\n",
      "132/132 [==============================] - 0s - loss: 0.9006 - acc: 0.6364     \n",
      "Epoch 11/50\n",
      "132/132 [==============================] - 0s - loss: 0.6999 - acc: 0.7500     \n",
      "Epoch 12/50\n",
      "132/132 [==============================] - 0s - loss: 0.6806 - acc: 0.7879     \n",
      "Epoch 13/50\n",
      "132/132 [==============================] - 0s - loss: 0.6973 - acc: 0.7273     \n",
      "Epoch 14/50\n",
      "132/132 [==============================] - 0s - loss: 0.4941 - acc: 0.8258     \n",
      "Epoch 15/50\n",
      "132/132 [==============================] - 0s - loss: 0.4708 - acc: 0.8182     \n",
      "Epoch 16/50\n",
      "132/132 [==============================] - 0s - loss: 0.3728 - acc: 0.9015     \n",
      "Epoch 17/50\n",
      "132/132 [==============================] - 0s - loss: 0.3364 - acc: 0.8788     \n",
      "Epoch 18/50\n",
      "132/132 [==============================] - 0s - loss: 0.3617 - acc: 0.8712     \n",
      "Epoch 19/50\n",
      "132/132 [==============================] - 0s - loss: 0.3695 - acc: 0.8561     \n",
      "Epoch 20/50\n",
      "132/132 [==============================] - 0s - loss: 0.5258 - acc: 0.8409     \n",
      "Epoch 21/50\n",
      "132/132 [==============================] - 0s - loss: 0.4963 - acc: 0.8030     \n",
      "Epoch 22/50\n",
      "132/132 [==============================] - 0s - loss: 0.3975 - acc: 0.8333     \n",
      "Epoch 23/50\n",
      "132/132 [==============================] - 0s - loss: 0.4166 - acc: 0.8485     \n",
      "Epoch 24/50\n",
      "132/132 [==============================] - 0s - loss: 0.3083 - acc: 0.9091     \n",
      "Epoch 25/50\n",
      "132/132 [==============================] - 0s - loss: 0.4042 - acc: 0.8485     \n",
      "Epoch 26/50\n",
      "132/132 [==============================] - 0s - loss: 0.3120 - acc: 0.8939     \n",
      "Epoch 27/50\n",
      "132/132 [==============================] - 0s - loss: 0.2701 - acc: 0.8788     \n",
      "Epoch 28/50\n",
      "132/132 [==============================] - 0s - loss: 0.2229 - acc: 0.9242     \n",
      "Epoch 29/50\n",
      "132/132 [==============================] - 0s - loss: 0.2158 - acc: 0.9242     \n",
      "Epoch 30/50\n",
      "132/132 [==============================] - 0s - loss: 0.1785 - acc: 0.9470     \n",
      "Epoch 31/50\n",
      "132/132 [==============================] - 0s - loss: 0.2076 - acc: 0.9318     \n",
      "Epoch 32/50\n",
      "132/132 [==============================] - 0s - loss: 0.1375 - acc: 0.9545     \n",
      "Epoch 33/50\n",
      "132/132 [==============================] - 0s - loss: 0.1337 - acc: 0.9545     \n",
      "Epoch 34/50\n",
      "132/132 [==============================] - 0s - loss: 0.0902 - acc: 0.9697     \n",
      "Epoch 35/50\n",
      "132/132 [==============================] - 0s - loss: 0.0835 - acc: 0.9773     \n",
      "Epoch 36/50\n",
      "132/132 [==============================] - 0s - loss: 0.0966 - acc: 0.9697     \n",
      "Epoch 37/50\n",
      "132/132 [==============================] - 0s - loss: 0.0798 - acc: 0.9621     \n",
      "Epoch 38/50\n",
      "132/132 [==============================] - 0s - loss: 0.0517 - acc: 0.9848     \n",
      "Epoch 39/50\n",
      "132/132 [==============================] - 0s - loss: 0.0326 - acc: 0.9924     \n",
      "Epoch 40/50\n",
      "132/132 [==============================] - 0s - loss: 0.0360 - acc: 1.0000     \n",
      "Epoch 41/50\n",
      "132/132 [==============================] - 0s - loss: 0.0552 - acc: 0.9848     \n",
      "Epoch 42/50\n",
      "132/132 [==============================] - 0s - loss: 0.0939 - acc: 0.9697     \n",
      "Epoch 43/50\n",
      "132/132 [==============================] - 0s - loss: 0.4027 - acc: 0.8788     \n",
      "Epoch 44/50\n",
      "132/132 [==============================] - 0s - loss: 0.8704 - acc: 0.7727     \n",
      "Epoch 45/50\n",
      "132/132 [==============================] - 0s - loss: 0.6022 - acc: 0.8030     \n",
      "Epoch 46/50\n",
      "132/132 [==============================] - 0s - loss: 0.7956 - acc: 0.7348     \n",
      "Epoch 47/50\n",
      "132/132 [==============================] - 0s - loss: 0.4254 - acc: 0.8409     \n",
      "Epoch 48/50\n",
      "132/132 [==============================] - 0s - loss: 0.3754 - acc: 0.8409     \n",
      "Epoch 49/50\n",
      "132/132 [==============================] - 0s - loss: 0.2907 - acc: 0.9015     \n",
      "Epoch 50/50\n",
      "132/132 [==============================] - 0s - loss: 0.2175 - acc: 0.9470     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fcc569bec50>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_indices, Y_train_oh, epochs = 50, batch_size = 32, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/56 [================>.............] - ETA: 0s\n",
      "Test accuracy =  0.821428562914\n"
     ]
    }
   ],
   "source": [
    "X_test_indices = sentences_to_indices(X_test, word_to_index, max_len = maxLen)\n",
    "Y_test_oh = convert_to_one_hot(Y_test, C = 5)\n",
    "loss, acc = model.evaluate(X_test_indices, Y_test_oh)\n",
    "print()\n",
    "print(\"Test accuracy = \", acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected emoji:😄 prediction: she got me a nice present\t❤️\n",
      "Expected emoji:😞 prediction: This girl is messing with me\t❤️\n",
      "Expected emoji:😞 prediction: work is horrible\t😄\n",
      "Expected emoji:🍴 prediction: any suggestions for dinner\t😄\n",
      "Expected emoji:😄 prediction: you brighten my day\t❤️\n",
      "Expected emoji:😞 prediction: she is a bully\t❤️\n",
      "Expected emoji:😞 prediction: My life is so boring\t❤️\n",
      "Expected emoji:😄 prediction: will you be my valentine\t😞\n",
      "Expected emoji:😄 prediction: What you did was awesome\t😞\n",
      "Expected emoji:😞 prediction: go away\t⚾\n"
     ]
    }
   ],
   "source": [
    "# This code allows you to see the mislabelled examples\n",
    "C = 5\n",
    "y_test_oh = np.eye(C)[Y_test.reshape(-1)]\n",
    "X_test_indices = sentences_to_indices(X_test, word_to_index, maxLen)\n",
    "pred = model.predict(X_test_indices)\n",
    "for i in range(len(X_test)):\n",
    "    x = X_test_indices\n",
    "    num = np.argmax(pred[i])\n",
    "    if(num != Y_test[i]):\n",
    "        print('Expected emoji:'+ label_to_emoji(Y_test[i]) + ' prediction: '+ X_test[i] + label_to_emoji(num).strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am not happy 😞\n"
     ]
    }
   ],
   "source": [
    "# Change the sentence below to see your prediction. Make sure all the words are in the Glove embeddings.  \n",
    "x_test = np.array(['I am not happy'])\n",
    "X_test_indices = sentences_to_indices(x_test, word_to_index, maxLen)\n",
    "print(x_test[0] +' '+  label_to_emoji(np.argmax(model.predict(X_test_indices))))"
   ]
  }
 ],
 "metadata": {
  "coursera": {
   "course_slug": "nlp-sequence-models",
   "graded_item_id": "RNnEs",
   "launcher_item_id": "acNYU"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}