Hierarchical Attention Networks .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yzhao/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.2\n",
    "set_session(tf.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import Input\n",
    "import keras\n",
    "from keras.optimizers import Adam\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "from keras.layers import *\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers.recurrent import LSTM\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LENGTH = 100\n",
    "MAX_SENTS = 15\n",
    "MAX_NB_WORDS = 20000\n",
    "EMBEDDING_DIM = 100\n",
    "VALIDATION_SPLIT = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25000, 3)\n",
      "Total 88582 unique tokens.\n",
      "Shape of data tensor: (25000, 15, 100)\n",
      "Shape of label tensor: (25000, 2)\n",
      "Number of positive and negative reviews in traing and validation set\n",
      "[ 9964. 10036.]\n",
      "[2536. 2464.]\n"
     ]
    }
   ],
   "source": [
    "data_train = pd.read_csv('labeledTrainData.tsv', sep='\\t')\n",
    "print(data_train.shape)\n",
    "\n",
    "from nltk import tokenize\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "reviews = []\n",
    "labels = []\n",
    "texts = []\n",
    "\n",
    "for idx in range(data_train.review.shape[0]):\n",
    "    text = data_train.review[idx]\n",
    "    texts.append(text)\n",
    "    sentences = tokenize.sent_tokenize(text)\n",
    "    reviews.append(sentences)\n",
    "    \n",
    "    labels.append(data_train.sentiment[idx])\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "\n",
    "data = np.zeros((len(texts), MAX_SENTS, MAX_SENT_LENGTH), dtype='int32')\n",
    "\n",
    "for i, sentences in enumerate(reviews):\n",
    "    for j, sent in enumerate(sentences):\n",
    "        if j< MAX_SENTS:\n",
    "            wordTokens = text_to_word_sequence(sent)\n",
    "            k=0\n",
    "            for _, word in enumerate(wordTokens):\n",
    "                if k<MAX_SENT_LENGTH and tokenizer.word_index[word]<MAX_NB_WORDS:\n",
    "                    data[i,j,k] = tokenizer.word_index[word]\n",
    "                    k=k+1                    \n",
    "                    \n",
    "word_index = tokenizer.word_index\n",
    "print('Total %s unique tokens.' % len(word_index))\n",
    "\n",
    "labels = to_categorical(np.asarray(labels))\n",
    "print('Shape of data tensor:', data.shape)\n",
    "print('Shape of label tensor:', labels.shape)\n",
    "\n",
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]\n",
    "nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
    "\n",
    "x_train = data[:-nb_validation_samples]\n",
    "y_train = labels[:-nb_validation_samples]\n",
    "x_val = data[-nb_validation_samples:]\n",
    "y_val = labels[-nb_validation_samples:]\n",
    "\n",
    "print('Number of positive and negative reviews in traing and validation set')\n",
    "print(y_train.sum(axis=0))\n",
    "print(y_val.sum(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model fitting - Hierachical LSTM\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 15, 100)           0         \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, 15, 200)           9019100   \n",
      "_________________________________________________________________\n",
      "bidirectional_9 (Bidirection (None, 200)               240800    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 2)                 402       \n",
      "=================================================================\n",
      "Total params: 9,260,302\n",
      "Trainable params: 9,260,302\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            input_length=MAX_SENT_LENGTH)\n",
    "\n",
    "sentence_input = Input(shape=(MAX_SENT_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sentence_input)\n",
    "l_lstm = Bidirectional(LSTM(100))(embedded_sequences)\n",
    "sentEncoder = Model(sentence_input, l_lstm)\n",
    "\n",
    "review_input = Input(shape=(MAX_SENTS,MAX_SENT_LENGTH), dtype='int32')\n",
    "review_encoder = TimeDistributed(sentEncoder)(review_input)\n",
    "l_lstm_sent = Bidirectional(LSTM(100))(review_encoder)\n",
    "preds = Dense(2, activation='softmax')(l_lstm_sent)\n",
    "model = Model(review_input, preds)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "\n",
    "print(\"model fitting - Hierachical LSTM\")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AttentionLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(** kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape)==3\n",
    "        # W.shape = (time_steps, time_steps)\n",
    "        self.W = self.add_weight(name='att_weight', \n",
    "                                 shape=(input_shape[1], input_shape[1]),\n",
    "                                 initializer='uniform',\n",
    "                                 trainable=True)\n",
    "        super(AttentionLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, inputs, mask=None):\n",
    "        # inputs.shape = (batch_size, time_steps, seq_len)\n",
    "        x = K.permute_dimensions(inputs, (0, 2, 1))\n",
    "        # x.shape = (batch_size, seq_len, time_steps)\n",
    "        # general\n",
    "        a = K.softmax(K.tanh(K.dot(x, self.W)))\n",
    "        a = K.permute_dimensions(a, (0, 2, 1))\n",
    "        outputs = a * inputs\n",
    "        outputs = K.sum(outputs, axis=1)\n",
    "        return outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0], input_shape[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_16 (InputLayer)        (None, 15, 100)           0         \n",
      "_________________________________________________________________\n",
      "time_distributed_13 (TimeDis (None, 15, 200)           9029100   \n",
      "_________________________________________________________________\n",
      "bidirectional_15 (Bidirectio (None, 15, 200)           180600    \n",
      "_________________________________________________________________\n",
      "time_distributed_14 (TimeDis (None, 15, 200)           40200     \n",
      "_________________________________________________________________\n",
      "attention_layer_5 (Attention (None, 200)               225       \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 2)                 402       \n",
      "=================================================================\n",
      "Total params: 9,250,527\n",
      "Trainable params: 9,250,527\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "model fitting - Hierachical attention network\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/10\n",
      "20000/20000 [==============================] - 89s 4ms/step - loss: 0.3191 - acc: 0.8627 - val_loss: 0.2695 - val_acc: 0.8918\n",
      "Epoch 2/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.1893 - acc: 0.9315 - val_loss: 0.3488 - val_acc: 0.8734\n",
      "Epoch 3/10\n",
      "20000/20000 [==============================] - 101s 5ms/step - loss: 0.1479 - acc: 0.9469 - val_loss: 0.2527 - val_acc: 0.9052\n",
      "Epoch 4/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.1163 - acc: 0.9575 - val_loss: 0.3383 - val_acc: 0.8824\n",
      "Epoch 5/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.0903 - acc: 0.9687 - val_loss: 0.2788 - val_acc: 0.8920\n",
      "Epoch 6/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.0755 - acc: 0.9750 - val_loss: 0.3923 - val_acc: 0.8864\n",
      "Epoch 7/10\n",
      "20000/20000 [==============================] - 101s 5ms/step - loss: 0.0509 - acc: 0.9837 - val_loss: 0.3715 - val_acc: 0.8812\n",
      "Epoch 8/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.0434 - acc: 0.9878 - val_loss: 0.4306 - val_acc: 0.8824\n",
      "Epoch 9/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.0244 - acc: 0.9929 - val_loss: 0.5162 - val_acc: 0.8876\n",
      "Epoch 10/10\n",
      "20000/20000 [==============================] - 102s 5ms/step - loss: 0.0147 - acc: 0.9952 - val_loss: 0.7390 - val_acc: 0.8848\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fed0349f128>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_input = Input(shape=(MAX_SENT_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sentence_input)\n",
    "l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)\n",
    "l_dense = TimeDistributed(Dense(200))(l_lstm)\n",
    "l_att = AttentionLayer()(l_dense)\n",
    "sentEncoder = Model(sentence_input, l_att)\n",
    "\n",
    "review_input = Input(shape=(MAX_SENTS,MAX_SENT_LENGTH), dtype='int32')\n",
    "review_encoder = TimeDistributed(sentEncoder)(review_input)\n",
    "l_lstm_sent = Bidirectional(GRU(100, return_sequences=True))(review_encoder)\n",
    "l_dense_sent = TimeDistributed(Dense(200))(l_lstm_sent)\n",
    "l_att_sent = AttentionLayer()(l_dense_sent)\n",
    "preds = Dense(2, activation='softmax')(l_att_sent)\n",
    "model = Model(review_input, preds)\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "\n",
    "print(\"model fitting - Hierachical attention network\")\n",
    "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=10, batch_size=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}