5. Recurrent Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recurrent Neural Networks\n",
    "\n",
    "Classical neural networks, including convolutional ones, suffer from two severe limitations:\n",
    "\n",
    "+ They only accept a fixed-sized vector as input and produce a fixed-sized vector as output.\n",
    "+ They do not consider the sequential nature of some data (language, video frames, time series, etc.)\n",
    "\n",
    "Recurrent neural networks overcome these limitations by allowing to operate over sequences of vectors (in the input, in the output, or both).\n",
    "\n",
    "RNN can be interpreted as running a fixed program (consisting of a recurrent transformation that can be applied as many times as we like) with certain inputs and certain internal variables."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning to add\n",
    "\n",
    "Source: http://projects.rajivshah.com/blog/2016/04/05/rnn_addition/ \n",
    "\n",
    "The objective of this code developed by Rajiv Shah is to train a RNN for adding a sequence of integers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import basic libraries\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.models.rnn import rnn_cell\n",
    "from tensorflow.models.rnn import rnn\n",
    "from tensorflow.models.rnn import seq2seq\n",
    "from numpy import sum\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import *\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will define first a set of hyperparameters, beign the most important ``num_units``, that is the parameter that represents the internal memory in the basic LSTM cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_units = 50  \n",
    "input_size = 1      \n",
    "batch_size = 50    \n",
    "seq_len = 15\n",
    "drop_out = 0.6 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we can write an auxiliar function to generate random sequences of integers (and the result of their addition):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Creates our random sequences\n",
    "def gen_data(min_length=5, max_length=15, n_batch=50):\n",
    "\n",
    "    X = np.concatenate([np.random.randint(10,size=(n_batch, max_length, 1))],\n",
    "                       axis=-1)\n",
    "    y = np.zeros((n_batch,))\n",
    "    # Compute masks and correct values\n",
    "    for n in range(n_batch):\n",
    "        # Randomly choose the sequence length\n",
    "        length = np.random.randint(min_length, max_length)\n",
    "        X[n, length:, 0] = 0\n",
    "        # Sum the dimensions of X to get the target value\n",
    "        y[n] = np.sum(X[n, :, 0]*1)\n",
    "    return (X,y)\n",
    "\n",
    "print gen_data(2,5,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are ready to star the model construction phase:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Model architecture\n",
    "num_layers = 2\n",
    "cell = rnn_cell.BasicLSTMCell(num_units)\n",
    "cell = rnn_cell.MultiRNNCell([cell] * num_layers)\n",
    "cell = rnn_cell.DropoutWrapper(cell,output_keep_prob=drop_out)\n",
    "\n",
    "# Create placeholders for X and y\n",
    "inputs = [tf.placeholder(tf.float32,shape=[batch_size,1]) for _ in range(seq_len)]\n",
    "result = tf.placeholder(tf.float32, shape=[batch_size])\n",
    "\n",
    "# We initialize the initial cell state to 0\n",
    "initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "\n",
    "outputs, states = seq2seq.rnn_decoder(inputs, initial_state, cell, scope ='rnnln')\n",
    "# We are only interested in the final LSTM output value\n",
    "outputs2 = outputs[-1]\n",
    "\n",
    "# Tranformation of the final LSTM output value to a real value\n",
    "W_o = tf.Variable(tf.random_normal([num_units,input_size], stddev=0.01))     \n",
    "b_o = tf.Variable(tf.random_normal([input_size], stddev=0.01))\n",
    "outputs3 = tf.matmul(outputs2, W_o) + b_o\n",
    "\n",
    "# Definition of the mean square loss function\n",
    "cost = tf.pow(tf.sub(tf.reshape(outputs3, [-1]), result),2)\n",
    "train_op = tf.train.RMSPropOptimizer(0.005, 0.2).minimize(cost) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Generate Validation Data\n",
    "tempX,y_val = gen_data(5,seq_len,batch_size)\n",
    "X_val = []\n",
    "for i in range(seq_len):\n",
    "    X_val.append(tempX[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##Session\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "train_score =[]\n",
    "val_score= []\n",
    "x_axis=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_epochs=1000\n",
    " \n",
    "for k in tqdm(range(1,num_epochs)):\n",
    "\n",
    "    #Generate Data for each epoch\n",
    "    tempX,y = gen_data(5,seq_len,batch_size)\n",
    "    X = []\n",
    "    for i in range(seq_len):\n",
    "        X.append(tempX[:,i,:])\n",
    "\n",
    "    #Create the dictionary of inputs to feed into sess.run\n",
    "    temp_dict = {inputs[i]:X[i] for i in range(seq_len)}\n",
    "    temp_dict.update({result: y})\n",
    "\n",
    "    _,c_train = sess.run([train_op,cost],feed_dict=temp_dict)   #perform an update on the parameters\n",
    "\n",
    "    val_dict = {inputs[i]:X_val[i] for i in range(seq_len)}  #create validation dictionary\n",
    "    val_dict.update({result: y_val})\n",
    "    c_val = sess.run([cost],feed_dict = val_dict )            #compute the cost on the validation set\n",
    "    if (k%100==0):\n",
    "        train_score.append(sum(c_train))\n",
    "        val_score.append(sum(c_val))\n",
    "        x_axis.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print \"Final Train cost: {}, on Epoch {}\".format(train_score[-1],k)\n",
    "print \"Final Validation cost: {}, on Epoch {}\".format(val_score[-1],k)\n",
    "plt.plot(train_score, 'r-', val_score, 'b-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##This part generates a new validation set to test against\n",
    "val_score_v =[]\n",
    "num_epochs=1\n",
    "\n",
    "for k in range(num_epochs):\n",
    "\n",
    "    #Generate Data for each epoch\n",
    "    tempX,y = gen_data(5,seq_len,batch_size)\n",
    "    X = []\n",
    "    for i in range(seq_len):\n",
    "        X.append(tempX[:,i,:])\n",
    "\n",
    "    val_dict = {inputs[i]:X[i] for i in range(seq_len)}\n",
    "    val_dict.update({result: y})\n",
    "    outv, c_val = sess.run([outputs3,cost],feed_dict = val_dict ) \n",
    "    val_score_v.append([c_val])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##Target\n",
    "tempX[3],y[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Prediction\n",
    "outv[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example\n",
    "\n",
    "A Recurrent Neural Network (LSTM) implementation example using TensorFlow library.\n",
    "\n",
    "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n",
    "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n",
    "\n",
    "Author: Aymeric Damien\n",
    "Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.models.rnn import rnn, rnn_cell\n",
    "import numpy as np\n",
    "\n",
    "# Import MINST data\n",
    "import sys\n",
    "sys.path.insert(0, './helpers')\n",
    "import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To classify images using a recurrent neural network, we consider every image row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then handle 28 sequences of 28 steps for every sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "\n",
    "display_step = 10\n",
    "\n",
    "# Network Parameters\n",
    "n_input = 28 # number of sequences for every sample\n",
    "n_steps = 28 # number of timesteps for every sequence\n",
    "n_hidden = 64 # hidden layer num of features\n",
    "n_classes = 10 # total classes (0-9 digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n",
    "\n",
    "# Tensorflow LSTM cell requires 2x n_hidden length (state & cell)\n",
    "istate = tf.placeholder(\"float\", [None, 2*n_hidden])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "# Define weights\n",
    "weights = {\n",
    "    'hidden': tf.Variable(tf.random_normal([n_input, n_hidden])), \n",
    "    # Hidden layer weights\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'hidden': tf.Variable(tf.random_normal([n_hidden])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def RNN(_X, _istate, _weights, _biases):\n",
    "    \n",
    "    # Define a lstm cell with tensorflow\n",
    "    lstm_cell = rnn_cell.BasicLSTMCell(n_hidden)\n",
    "\n",
    "    # input shape: (batch_size, n_steps, n_input)\n",
    "    _X = tf.transpose(_X, [1, 0, 2])  # permute n_steps and batch_size\n",
    "        \n",
    "    # Reshape to prepare input to hidden activation\n",
    "    _X = tf.reshape(_X, [-1, n_input]) # (n_steps*batch_size, n_input)\n",
    "    # Linear activation\n",
    "    _X = tf.matmul(_X, _weights['hidden']) + _biases['hidden']\n",
    "    \n",
    "    # Split data because rnn cell needs a list of inputs \n",
    "    # for the RNN inner loop\n",
    "    _X = tf.split(0, n_steps, _X) # n_steps * (batch_size, n_hidden)\n",
    "\n",
    "    # Get lstm cell output\n",
    "    outputs, states = rnn.rnn(lstm_cell, _X, initial_state=_istate)\n",
    "\n",
    "    # Linear activation\n",
    "    # Get inner loop last output\n",
    "    return tf.matmul(outputs[-1], _weights['out']) + _biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred = RNN(x, istate, weights, biases)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # Softmax loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# Initializing the variables\n",
    "init = tf.initialize_all_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        # mnist.train is a tensor (an n-dimensional array) \n",
    "        # with a shape of [55000, 784]\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        # Reshape data to get 28 seq of 28 elements\n",
    "        batch_xs = batch_xs.reshape((batch_size, n_steps, n_input))\n",
    "        # Fit training using batch data\n",
    "        sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys,\n",
    "                                       istate: np.zeros((batch_size, 2*n_hidden))})\n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch accuracy\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys,\n",
    "                                                istate: np.zeros((batch_size, 2*n_hidden))})\n",
    "            # Calculate batch loss\n",
    "            loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys,\n",
    "                                             istate: np.zeros((batch_size, 2*n_hidden))})\n",
    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Training Accuracy= \" + \"{:.5f}\".format(acc)\n",
    "        step += 1\n",
    "    print \"Optimization Finished!\"\n",
    "    # Calculate accuracy for mnist test images\n",
    "    test_len = 5000\n",
    "    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n",
    "    test_label = mnist.test.labels[:test_len]\n",
    "    print \"Testing Accuracy:\", sess.run(accuracy, feed_dict={x: test_data, y: test_label,\n",
    "                                                             istate: np.zeros((test_len, 2*n_hidden))})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tflearn\n",
    "\n",
    "def textfile_to_seq(file, seq_maxlen=25, redun_step=3):\n",
    "    \"\"\" string_to_semi_redundant_sequences.\n",
    "    Vectorize a string and returns parsed sequences and targets, along with\n",
    "    the associated dictionary.\n",
    "    Arguments:\n",
    "        string: `str`. Lower-case text from input text file.\n",
    "        seq_maxlen: `int`. Maximum length of a sequence. Default: 25.\n",
    "        redun_step: `int`. Redundancy step. Default: 3.\n",
    "    Returns:\n",
    "        `tuple`: (inputs, targets, dictionary)\n",
    "    \"\"\"\n",
    "    import numpy as np\n",
    "    import re\n",
    "    print(\"Vectorizing text...\")\n",
    "    \n",
    "    import codecs\n",
    "    f = codecs.open('toponims.txt', \"r\", \"utf-8\")\n",
    "    string = f.read()\n",
    "    string.encode('utf-8')\n",
    "    string = re.sub( '([A-Z])', '^\\\\1', string ).lower()\n",
    "    chars = set()\n",
    "    chars.update(string)\n",
    "    char_idx = {c: i for i, c in enumerate(chars)}\n",
    "\n",
    "    sequences = []\n",
    "    next_chars = []\n",
    "    for i in range(0, len(string) - seq_maxlen, redun_step):\n",
    "        sequences.append(string[i: i + seq_maxlen])\n",
    "        next_chars.append(string[i + seq_maxlen])\n",
    "\n",
    "    X = np.zeros((len(sequences), seq_maxlen, len(chars)), dtype=np.bool)\n",
    "    Y = np.zeros((len(sequences), len(chars)), dtype=np.bool)\n",
    "    for i, seq in enumerate(sequences):\n",
    "        for t, char in enumerate(seq):\n",
    "            X[i, t, char_idx[char]] = 1\n",
    "        Y[i, char_idx[next_chars[i]]] = 1\n",
    "\n",
    "    print(\"Text total length: \" + str(len(string)))\n",
    "    print(\"Distinct chars: \" + str(len(chars)))\n",
    "    print(\"Total sequences: \" + str(len(sequences)))\n",
    "    return X, Y, char_idx\n",
    "\n",
    "def random_sequence_from_string(string, seq_maxlen):\n",
    "    import random\n",
    "    rand_index = random.randint(0, len(string) - seq_maxlen - 1)\n",
    "    return string[rand_index: rand_index + seq_maxlen]\n",
    "\n",
    "def random_sequence_from_textfile(path, seq_maxlen):\n",
    "    import codecs\n",
    "    import re\n",
    "    f = codecs.open(path, \"r\", \"utf-8\")\n",
    "    text = f.read()\n",
    "    text.encode('utf-8')\n",
    "    text = re.sub( '([A-Z])', '^\\\\1', text ).lower()\n",
    "    return random_sequence_from_string(text, seq_maxlen)\n",
    "\n",
    "path = 'toponims.txt'\n",
    "maxlen = 20\n",
    "\n",
    "X, Y, char_idx = \\\n",
    "    textfile_to_seq(path, seq_maxlen=maxlen, redun_step=2)\n",
    "\n",
    "g = tflearn.input_data(shape=[None, maxlen, len(char_idx)])\n",
    "g = tflearn.lstm(g, 64, return_seq=True)\n",
    "g = tflearn.dropout(g, 0.5)\n",
    "g = tflearn.lstm(g, 64)\n",
    "g = tflearn.dropout(g, 0.5)\n",
    "g = tflearn.fully_connected(g, len(char_idx), activation='softmax')\n",
    "g = tflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',\n",
    "                       learning_rate=0.01)\n",
    "\n",
    "m = tflearn.SequenceGenerator(g, dictionary=char_idx,\n",
    "                              seq_maxlen=maxlen,\n",
    "                              clip_gradients=5.0)\n",
    "\n",
    "for i in range(100):\n",
    "    seed = random_sequence_from_textfile(path, maxlen)\n",
    "    m.fit(X, Y, validation_set=0.1, batch_size=128,\n",
    "          n_epoch=1, run_id='toponims')\n",
    "    print(\"-- TESTING...\")\n",
    "    print(\"-- EPOCH = \", i)\n",
    "    print(\"-- Test with temperature of 1.2 --\")\n",
    "    print(m.generate(30, temperature=1.2, seq_seed=seed))\n",
    "    print(\"-- Test with temperature of 1.0 --\")\n",
    "    print(m.generate(30, temperature=1.0, seq_seed=seed))\n",
    "    print(\"-- Test with temperature of 0.5 --\")\n",
    "    print(m.generate(30, temperature=0.5, seq_seed=seed))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "widgets": {
   "state": {},
   "version": "1.1.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}