neural_machine_translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/shivanikohli/Documents/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# dependencies\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import pickle\n",
    "import data_utils\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import pandas as pd\n",
    "from nltk.translate.bleu_score import sentence_bleu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def read_sentences(file_path):\n",
    "\tsentences = []\n",
    "\n",
    "\twith open(file_path, 'r') as reader:\n",
    "\t\tfor s in reader:\n",
    "\t\t\tsentences.append(s.strip())\n",
    "\n",
    "\treturn sentences\n",
    "\n",
    "def iteritems(dic):\n",
    "    return iter([(key, dic[key]) for key in dic])\n",
    "\n",
    "def create_dataset(en_sentences, sn_sentences):\n",
    "\n",
    "\ten_vocab_dict = Counter(word.strip(',.\" ;:)(][?!') for sentence in en_sentences for word in sentence.split())\n",
    "\tsn_vocab_dict = Counter(word.strip(',.\" ;:)(][?!') for sentence in sn_sentences for word in sentence.split())\n",
    "\n",
    "\ten_vocab = list(map(lambda x: x[0], sorted(en_vocab_dict.items(), key = lambda x: -x[1])))\n",
    "\tsn_vocab = list(map(lambda x: x[0], sorted(sn_vocab_dict.items(), key = lambda x: -x[1])))\n",
    "\n",
    "\ten_vocab = en_vocab[:20000]\n",
    "\tsn_vocab = sn_vocab[:30000]\n",
    "\n",
    "\tstart_idx = 2\n",
    "\ten_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(en_vocab)])\n",
    "\ten_word2idx['<ukn>'] = 0\n",
    "\ten_word2idx['<pad>'] = 1\n",
    "\n",
    "\ten_idx2word = dict([(idx, word) for word, idx in iteritems(en_word2idx)])\n",
    "\n",
    "\n",
    "\tstart_idx = 4\n",
    "\tsn_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(sn_vocab)])\n",
    "\tsn_word2idx['<ukn>'] = 0\n",
    "\tsn_word2idx['<go>']  = 1\n",
    "\tsn_word2idx['<eos>'] = 2\n",
    "\tsn_word2idx['<pad>'] = 3\n",
    "\n",
    "\tsn_idx2word = dict([(idx, word) for word, idx in iteritems(sn_word2idx)])\n",
    "\n",
    "\tx = [[en_word2idx.get(word.strip(',.\" ;:)(][?!'), 0) for word in sentence.split()] for sentence in en_sentences]\n",
    "\ty = [[sn_word2idx.get(word.strip(',.\" ;:)(][?!'), 0) for word in sentence.split()] for sentence in sn_sentences]\n",
    "\n",
    "\tX = []\n",
    "\tY = []\n",
    "\tfor i in range(len(x)):\n",
    "\t\tn1 = len(x[i])\n",
    "\t\tn2 = len(y[i])\n",
    "\t\tn = n1 if n1 < n2 else n2 \n",
    "\t\tif abs(n1 - n2) <= 0.3 * n:\n",
    "\t\t\tif n1 <= 15 and n2 <= 15:\n",
    "\t\t\t\tX.append(x[i])\n",
    "\t\t\t\tY.append(y[i])\n",
    "\n",
    "\treturn X, Y, en_word2idx, en_idx2word, en_vocab, sn_word2idx, sn_idx2word, sn_vocab\n",
    "\n",
    "def save_dataset(file_path, obj):\n",
    "\twith open(file_path, 'wb') as f:\n",
    "\t\tpickle.dump(obj, f, -1)\n",
    "\n",
    "def main():\n",
    "    en_sentences = read_sentences('./Data/bible.en')\n",
    "    sn_sentences = read_sentences('./Data/bible.san')\n",
    "\n",
    "    save_dataset('./Data/bible2.pkl', create_dataset(sn_sentences, en_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_sanskrit(uni):\n",
    "    a = bytearray(uni, encoding = \"utf-8\").decode('unicode-escape')\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read dataset\n",
    "def read_dataset(file_path):\n",
    "    with open(file_path, 'rb') as f:\n",
    "        return pickle.load(f,encoding=\"utf_8\")\n",
    "\n",
    "X, Y, sn_word2idx, sn_idx2word, sn_vocab, en_word2idx, en_idx2word, en_vocab = read_dataset('./Data/data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence in Sanskrit - encoded: [688, 1181, 506, 6, 1181, 11328, 0]\n",
      "Sentence in English - encoded: [5, 583, 6, 5, 462, 6, 39, 139, 0, 5, 62, 6, 333, 0, 5, 62, 6, 328, 0]\n",
      "Decoded:\n",
      "------------------------\n",
      "तस्य पुत्रो ऽम्मीनादब् तस्य पुत्रो नहशोन् तस्य पुत्रः सल्मोन्। \n",
      "\n",
      "and aram begat aminadab <ukn> and aminadab begat naasson <ukn> and naasson begat salmon <ukn> "
     ]
    }
   ],
   "source": [
    "#inspecting data\n",
    "print('Sentence in Sanskrit - encoded:', X[0])\n",
    "print('Sentence in English - encoded:', Y[0])\n",
    "print('Decoded:\\n------------------------')\n",
    "\n",
    "for i in range(len(X[3])):\n",
    "    print(convert_sanskrit(sn_idx2word[X[3][i]]), end = \" \")\n",
    "    \n",
    "print('\\n')\n",
    "\n",
    "for i in range(len(Y[3])):\n",
    "    print(en_idx2word[Y[3][i]], end = \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.58403468502933\n",
      "21.631981637337415\n"
     ]
    }
   ],
   "source": [
    "print(sum([len(sentence) for sentence in X]) / len(X))\n",
    "print(sum([len(sentence) for sentence in Y]) / len(Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data processing\n",
    "\n",
    "# data padding\n",
    "def data_padding(x, y, length = 10):\n",
    "    for i in range(len(x)):\n",
    "        x[i] = x[i] + (length - len(x[i])) * [sn_word2idx['<pad>']]\n",
    "        y[i] = [en_word2idx['<go>']] + y[i] + [en_word2idx['<eos>']] + (length-len(y[i])) * [en_word2idx['<pad>']]\n",
    "\n",
    "data_padding(X, Y)\n",
    "\n",
    "# data splitting\n",
    "X_train,  X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.1)\n",
    "\n",
    "del X\n",
    "del Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_seq_len = 10\n",
    "output_seq_len = 12\n",
    "sn_vocab_size = len(sn_vocab) + 2 # + <pad>, <ukn>\n",
    "en_vocab_size = len(en_vocab) + 4 # + <pad>, <ukn>, <eos>, <go>\n",
    "\n",
    "# placeholders\n",
    "encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "targets = [decoder_inputs[i+1] for i in range(output_seq_len-1)]\n",
    "# add one more target\n",
    "targets.append(tf.placeholder(dtype = tf.int32, shape = [None], name = 'last_target'))\n",
    "target_weights = [tf.placeholder(dtype = tf.float32, shape = [None], name = 'target_w{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "# output projection\n",
    "size = 512\n",
    "w_t = tf.get_variable('proj_w', [en_vocab_size, size], tf.float32)\n",
    "b = tf.get_variable('proj_b', [en_vocab_size], tf.float32)\n",
    "w = tf.transpose(w_t)\n",
    "output_projection = (w, b)\n",
    "\n",
    "outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                            encoder_inputs,\n",
    "                                            decoder_inputs,\n",
    "                                            tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                            num_encoder_symbols = sn_vocab_size,\n",
    "                                            num_decoder_symbols = en_vocab_size,\n",
    "                                            embedding_size = 100,\n",
    "                                            feed_previous = False,\n",
    "                                            output_projection = output_projection,\n",
    "                                            dtype = tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/shivanikohli/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:1346: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# define our loss function\n",
    "\n",
    "def sampled_loss(labels, logits):\n",
    "    return tf.nn.sampled_softmax_loss(\n",
    "                        weights = w_t,\n",
    "                        biases = b,\n",
    "                        labels = tf.reshape(labels, [-1, 1]),\n",
    "                        inputs = logits,\n",
    "                        num_sampled = 512,\n",
    "                        num_classes = en_vocab_size)\n",
    "\n",
    "# Weighted cross-entropy loss for a sequence of logits\n",
    "loss = tf.contrib.legacy_seq2seq.sequence_loss(outputs, targets, target_weights, softmax_loss_function = sampled_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's define some helper functions\n",
    "\n",
    "# simple softmax function\n",
    "def softmax(x):\n",
    "    n = np.max(x)\n",
    "    e_x = np.exp(x - n)\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "# feed data into placeholders\n",
    "def feed_dict(x, y, batch_size = 64):\n",
    "    feed = {}\n",
    "    \n",
    "    idxes = np.random.choice(len(x), size = batch_size, replace = False)\n",
    "    \n",
    "    for i in range(input_seq_len):\n",
    "        feed[encoder_inputs[i].name] = np.array([x[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    for i in range(output_seq_len):\n",
    "        feed[decoder_inputs[i].name] = np.array([y[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    feed[targets[len(targets)-1].name] = np.full(shape = [batch_size], fill_value = en_word2idx['<pad>'], dtype = np.int32)\n",
    "    \n",
    "    for i in range(output_seq_len-1):\n",
    "        batch_weights = np.ones(batch_size, dtype = np.float32)\n",
    "        target = feed[decoder_inputs[i+1].name]\n",
    "        for j in range(batch_size):\n",
    "            if target[j] == en_word2idx['<pad>']:\n",
    "                batch_weights[j] = 0.0\n",
    "        feed[target_weights[i].name] = batch_weights\n",
    "        \n",
    "    feed[target_weights[output_seq_len-1].name] = np.zeros(batch_size, dtype = np.float32)\n",
    "    \n",
    "    return feed\n",
    "\n",
    "# decode output sequence\n",
    "def decode_output(output_seq):\n",
    "    words = []\n",
    "    for i in range(output_seq_len):\n",
    "        smax = softmax(output_seq[i])\n",
    "        idx = np.argmax(smax)\n",
    "        words.append(de_idx2word[idx])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ops and hyperparameters\n",
    "learning_rate = 0.002\n",
    "batch_size = 512\n",
    "steps = 20000\n",
    "\n",
    "# ops for projecting outputs\n",
    "outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "# training op\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "# init op\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# forward step\n",
    "def forward_step(sess, feed):\n",
    "    output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "    return output_sequences\n",
    "\n",
    "# training step\n",
    "def backward_step(sess, feed):\n",
    "    sess.run(optimizer, feed_dict = feed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will use this list to plot losses through steps\n",
    "losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------TRAINING------------------\n",
      "Running from scratch: generating random model parameters.\n",
      "step: 0, loss: 8.284358978271484\n",
      "step: 4, loss: 8.286918640136719\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-d7c5287ffead>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m         \u001b[0mbackward_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-30-eab4e0fa9754>\u001b[0m in \u001b[0;36mbackward_step\u001b[0;34m(sess, feed)\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m# training step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbackward_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1359\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# let's train the model\n",
    "\n",
    "# save a checkpoint so we can restore the model later \n",
    "\n",
    "# checkpointsPath = './checkpoints501previous/'\n",
    "restore = False\n",
    "starting_step = 0\n",
    "\n",
    "print('------------------TRAINING------------------')\n",
    "with tf.Session() as sess:\n",
    "    if (restore):\n",
    "        print('Restoring')\n",
    "        with open(checkpointsPath + 'checkpoint') as f:\n",
    "            starting_step = int(re.match('model_checkpoint_path: \"-([0-9]+)\"', list(f)[0]).groups()[0]) + 1\n",
    "        saver.restore(sess, tf.train.latest_checkpoint(checkpointsPath))\n",
    "        print('Running from step {}'.format(starting_step))\n",
    "    else:\n",
    "        print('Running from scratch: generating random model parameters.')\n",
    "        sess.run(init)\n",
    "    \n",
    "    t = time.time()\n",
    "    for step in range(starting_step, starting_step + steps):\n",
    "        feed = feed_dict(X_train, Y_train)\n",
    "            \n",
    "        backward_step(sess, feed)\n",
    "        \n",
    "        if step % 5 == 4 or step == 0:\n",
    "            loss_value = sess.run(loss, feed_dict = feed)\n",
    "            print('step: {}, loss: {}'.format(step, loss_value))\n",
    "            losses.append(loss_value)\n",
    "        \n",
    "        if step % 20 == 19:\n",
    "            saver.save(sess, checkpointsPath, global_step=step)\n",
    "            print('Checkpoint is saved')\n",
    "            \n",
    "    print('Training time for {} steps: {}s'.format(steps, time.time() - t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'tensorflow' has no attribute 'restore'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-54403420fb75>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: module 'tensorflow' has no attribute 'restore'"
     ]
    }
   ],
   "source": [
    "tf.restore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEfCAYAAAAUfVINAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4jFf/BvB7ZrKSxERC9iCLJZSgJBSxZEF+GpES2pcmoUJbfelLrVWUxl7UWkmsaUtDETR2EVujllpaBI09QSSpRPaZ3x+RqUkim5k8M3J/rstVsz3zPROdO+c85zlHlJ6eLgcREZGGEQtdABERUVkYUEREpJEYUEREpJEYUEREpJEYUEREpJEYUEREpJEYUEREpJEYUEREpJFqVUAlJiYKXYJasX3aje3Tbmyf6tWqgCIiIu3BgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo3EgCIiIo0kWECFhYVBKpUq/WnatKlQ5RARkYbREfLNnZ2dsXv3bsVtiUQiYDVERKRJBA0oHR0dWFhYCFkCERFpKEHPQSUlJaFFixZo3bo1QkJCkJSUJGQ5RESkQUTp6elyId74wIEDyMzMhLOzM548eYIFCxYgMTERp0+fRv369V/5usTExBqskoiI1MXZ2bncxwULqJIyMzPh6uqKsWPH4tNPP1XLeyQmJlb4gWgztk+7sX3aje1TPY2ZZm5kZITmzZvj1q1bQpdCREQaQGMCKicnB4mJiZw0QUREAAScxTdt2jT07t0btra2inNQz58/x5AhQ4QqiYiINIhgAfXgwQOMGDECqampMDc3x9tvv40DBw7A3t5eqJKIiEiDCBZQkZGRQr01ERFpAY05B0VERPQyBhQREWkkBhQREWkkBhQREWkkBhQREWkkBhQREWmkWhNQH8U9RcQdQXcXISKiKqg1AfXzrWzsfsSAIiLSFrUmoABALBK6AiIiqqxaFVDcUJ6ISHvUqoASsQdFRKQ1alVASRhQRERao3YFlNAFEBFRpdWqgBKLNGJ3eyIiqoRaFVAc4iMi0h61KqA4zZyISHvUroASugAiIqq0WvWdzSE+IiLtUasCqq6EkySIiLRFrQmoeW71kJhVa5pLRKT1atU39oNcMWRy9qKIiLRBrQmoghe5lC8Ttg4iIqqcWhNQMllRQuXL2IMiItIGtSegXvy3gD0oIiKtUGsCqvBFx6mA56CIiLRCrQmoAsUQn8CFEBFRpdSagFL0oHgOiohIK9SagJIpAkrYOoiIqHJqUUBxFh8RkTbRmIBatGgRpFIpJkyYoJbjF/I6KCIiraIRAXXmzBls2LABLVu2VNt7FA/tcRYfEZF2EDygMjIy8NFHH+G7776DVCpV2/vIUBRMPAdFRKQdBA+osWPHws/PDx4eHmp9n+ZSXQA8B0VEpC10hHzzDRs24NatW1izZk2lX5OYmFit93IH4Gqij6S792D2z5vbjaru56Mt2D7txvZpN1W3z9nZudzHBQuoxMREzJo1C7/++iv09PQq/bqKGlQenct3YGltA2drg2ofQ5MlJia+1uej6dg+7cb2aTch2idYQCUkJCA1NRWdOnVS3FdYWIiTJ08iMjISDx48gL6+vkrf01AiR2Y+h/iIiLSBYAHl6+uLtm3bKt33ySefwNHREZ9//nmVelWVZaoLpOa8ucN7RERvEsECSiqVlpq1V6dOHZiamsLFxUU976krxxMGFBGRVhB8Fl9NkurI8TSXAUVEpA0EncVX0p49e9R6fD0xkFHIc1BERNqgVvWgJCKuJEFEpC1qVUDpiORci4+ISEvUsoDiShJERNqi1gUU1+IjItIOtSqgeA6KiEh71KqA0hEBO5NykMuZfEREGq92BZS4KJguPMkTuBIiIqpI7QooUdF/JWKRsIUQEVGFalVASUTK/yUiIs1VqwKquLGcJ0FEpPlqVUAVz43gtVBERJqvVgVUwYtcyuO1UEREGk9lAZWQkID9+/cjKytLVYdUuQJ50cmnPPagiIg0XpUDav78+fD391e6LzAwEL1798bgwYPRsWNH3LlzR2UFqpJUtyiY8ngdFBGRxqtyQO3YsUNpQ8G9e/di//79+O9//4vw8HDk5eVh/vz5Ki1SVdqYyNDDWp9DfEREWqDK+0Hdu3cPzs7OitsxMTFwdHTEV199BQBITEzE5s2bVVehitXXF3OSBBGRFqjWOajCwkLF3+Pi4tCrVy/FbWtrazx+/Pj1K1MTPYmIQ3xERFqgygHl5OSk2Pn24MGDSE5Ohqenp+Lx+/fvQyqVqq5CFdMTcxYfEZE2qPIQ35gxYzB8+HA0atQIz58/R9OmTdGjRw/F43FxcXjrrbdUWqQq6YnZgyIi0gZVDih/f3+Ymppi//79MDY2xvDhw6GjU3SYtLQ0mJmZITAwUOWFqoquhNPMiYi0QZUDCgC6d++O7t27l7rf1NRUoydIAEU9KG77TkSk+aoVUABw9+5dnDhxAo8fP4a/vz9sbW1RUFCAtLQ0mJqaKnpVmkZXLOJ+UEREWqBaKTJlyhR8//33KCwshEgkQuvWrWFra4vnz5+jXbt2mDRpEj755BNV16oS+hIRsrnvOxGRxqvyLL5ly5Zh1apV+OSTT7Bjxw7IX1oa3MTEBL6+vti9e7dKi1QlzuIjItIOVQ6oDRs2YNCgQZg5c2aZs/VatmyJmzdvqqQ4dcgtlOO7y5lCl0FERBWockDdu3cPnTt3fuXjxsbGyMjIeK2i1CmrgOefiIi0QZUDqn79+khOTn7l41euXIGVldVrFaVOxVPMCznVnIhIo1U5oLy9vbFhwwakpqaWeuyPP/7A5s2b4evrq5Li1CH3xSpNPA9FRKTZqhxQU6ZMgVgsRufOnTFjxgyIRCJERUUhJCQEXl5esLa2xoQJE9RRq0o8fzHEx4t1iYg0W5UDysLCAkePHkXv3r0RExMDuVyOn3/+GQcPHkRgYCD2799fqbX41q5di86dO8POzg52dnbw8vLCvn37qtWIqpjoagyA274TEWm6al0HZW5ujqVLl2Lp0qV48uQJZDIZzM3NIRZXPu+sra0xc+ZMODo6QiaT4ccff8QHH3yAo0ePolWrVtUpq1IaG+vAqo5YMdRHRESa6bW3fDc3N0fDhg3x6NEjXL16tdKv8/X1hZeXFxwcHODk5IQvv/wSRkZGOHPmzOuWVKGHz2UYsO+J2t+HiIiqr8oBtW7dOoSGhird97///Q8uLi7o3LkzunbtWuYEivIUFhZi27ZtyMrKQseOHataUrVcyyiokfchIqLqEaWnp1fpZEz37t3x9ttvY+HChQCAY8eOwc/PDwMHDoSLiwsWLlyIoUOHIiwsrMJjXblyBd7e3sjJyUHdunWxdu1a+Pj4lPuaxMTEqpRbpg7H6wAAznR5/trHIiKi6nl5d/ayVPkc1O3bt/Gf//xHcXvHjh2wsbHB6tWrIRaLkZGRgV9++aVSAeXs7Iz4+HhkZGRg165dGD16NHbv3g0XF5dyX1NdiYmJRa8/fv+1j6WJFO17Q7F92o3t025CtK/KAZWXlwddXV3F7SNHjsDT01MxQcLBwaHcC3lfpqenBwcHBwBA27Ztce7cOaxcuRLLly+vallERPSGqfI5qEaNGuHo0aMAgHPnziEpKQk9e/ZUPP7o0SMYGxtXqxiZTIa8vLxqvbYqFrjXg4+tvtrfh4iIqq/KPaiQkBBMmDAB165dw4MHD2BjYwMvLy/F46dPn0bz5s0rPM6MGTPg7e0NGxsbZGZmIjo6GsePH8fWrVurWlKVOZjoYN+9XDzLl8FY97UnMhIRkRpUOaBGjBgBPT097N+/H23atMHYsWNhaGgIoGjL98ePHyMkJKTC46SkpGDkyJF49OgRTExM0LJlS0RHR6NXr15Vb0UVFW8HlZkvh7Fu+c8lIiJhVOtC3WHDhmHYsGGl7jc1NVUM/1Vk1apV1XlrlTDSFQHgahJERJpMJfuy5+bmIiYmBunp6ejTpw9sbGxUcVi1ecdSH42NJcjj1u9ERBqryidgxo8fjy5duihuFxQUwMfHByNHjsSECRPg7u6OK1euqLRIdTCQiLiiORGRBqtyQMXFxSldTPvLL7/gjz/+wMKFC3HgwAGYmZlhwYIFKi1SHfTEIvagiIg0WJWH+B4+fIhGjRopbu/duxetWrVSTIwICQnB6tWrVVehmuhJuOUGEZEmq3IPSkdHB9nZ2QAAuVyOY8eOKc28k0qlePr0qeoqVBNd8b9DfB/Hp0G67j7kcgYWEZGmqHJAubi4YOvWrUhPT8fmzZuRlpYGT09PxeN37tyBubm5SotUB32JCPkvhvh+uFG0Jl9aLk9KERFpiioP8U2cOBGBgYGKJYrc3NyUJk3s27cP7dq1U12FaqInLr3t+8PnMtQ3kAhTEBERKalyQHl4eCAuLg5HjhyBsbExAgICFI+lpaWhS5cu8PX1VWmR6qArFiG3xCSJlOxCtASv3CUi0gTVug6qWbNmaNasWan7TU1NK7WKuSbQE4tKXah7858C9NTsS7iIiGqNal+o+/fff2P//v24c+cOAMDe3h7e3t5o0qSJyopTJwMdEZ4XKAfUhNMZ+KiFkUAVERHRy6oVUFOnTsXq1ashkymfxJkyZQpGjRqFOXPmqKQ4dTIqI6CIiEhzVHkW34oVK7By5Ur07dsX+/fvx+3bt3H79m3s378fvr6+WLVqFVauXKmOWlXKSFeELAYUEZHGqnJAbdy4Ed7e3ti0aRM6dOgAExMTmJiYoEOHDti4cSM8PT2xfv16NZSqWnV1xcjK57RyIiJNVeWASkpKgre39ysf9/b2xu3bt1+rqJpQV0eERRczkfSsQOhSiIioDFUOKFNTUyQmJr7y8Rs3bsDU1PS1iqoJdV9suXHkfq7S/QVc/oiISCNUOaD69u2LiIgIREVFKS0NJJfL8cMPPyAyMlIrroMyfhFQMigH0tV09qiIiDRBlWfxTZ8+HQkJCRgzZgxmzJgBR0dHAMCtW7fw+PFjtGrVCl9++aXKC1W1ujpF2VyywzT4YCouD7IUoCIiInpZlQNKKpXi8OHDWL9+vdJ1UK1bt4aPjw969+6Ne/fuQSqVqrxYVSoe4isZUNmc2UdEpBGqdR2Unp4eRo4ciZEjR5Z6bOHChfjmm280fkXzujpFAfXyakd/DrJE5x0pAlVEREQvq/I5qDeFnqQooHJeSijLOmJk5Mkh47YbRESCq70B9aLlLweUWCSCgUSkdB8REQmj1gaUo4kO/BsbKs45megV9agMuQQSEZFGqLUBJRKJ0LGhHp4XyCEWAXc+sAYA1NERISufAUVEJLRKTZI4e/ZspQ/44MGDahdT0wx1RMjKl+HF6SgARZMn2IMiIhJepQLK09MTIpGo4iei6ILdyj5XaPoSEbIL5UoBVUeXAUVEpAkqFVArVqxQdx2CMJAAz/PlEL8UqIYSBhQRkSaoVEC9//776q5DEPoSETIL5BziIyLSQLV2kgQAGEhEyH4xSaJY0RAft+EgIhJarQ+o5yUDSkfMjQyJiDSAYAG1ePFi9OjRA3Z2dnB0dERgYCD+/PPPGq2hOKAkL52DqqsjwnNOMyciEpxgAXX8+HEMHz4c+/btw65du6Cjo4P+/fsjLS2txmrQl4iQVSBT6kEZ6hTN7CMiImFVa7FYVdi+fbvS7TVr1sDe3h6nT59Gnz59aqSG4h5Uff1/E6qOjohDfEREGkBjzkFlZmZCJpPV6DYdBjoi5BZCaYjPQCJCLgOKiEhwovT0dI34Ng4KCsLNmzdx9OhRSCSSVz6vvO3mqyo9H/D6rQ4aG8rwc/scAEDUfR2k5IrwuUO+yt6HiIhKc3Z2LvdxwYb4XjZlyhScPn0asbGx5YYTUHGDypOYmKj0+sx8GfDbQzSSGsLZ2Q4AYFOQiWfpBXByqqc1K2IUK9m+Nw3bp93YPu0mRPsEH+KbPHkytm3bhl27dqFx48Y1+t4GL67Qtarz78egJxYh4moW5l14VqO1EBGRMkF7UBMnTsT27duxe/duNG3atMbfX+fF9D0j3X8Dqvivl59yiI+ISEiCBdT48eOxZcsWbN68GVKpFCkpRVut161bF0ZGRkKVpdhp11BHu4b3iIjeNIIN8YWHh+PZs2fw8/NDs2bNFH++++67Gq+lQPbvPBHdF70qAwkDiohISIL1oNLT04V661JenlVevBW8AXtQRESCEnyShCYoqwd1I6NAqHKIiAgaMs1cSOPbGKOPnYHidu6LZY6OPMgVqiQiIgIDCtPamSjd5jJHRESagUN8JVgY/vuRyOUMKyIioTCgSvCwNsClgRYAgDzuW0hEJBgGVBnsjIpGPi02PkD4X5kCV0NEVDsxoCrw26M8oUsgIqqVGFAV4FkoIiJhMKAqIGNCEREJggFVAU7kIyISBgPqFXb6mAMAZBzkIyISBAPqFZpLi2bysQdFRCQMBtQr1NUtWpOP56CIiITBgHqFOi9WMy9kQBERCYIB9QpiUVFAcVsoIiJhMKDKEdWzPrh2LBGRMBhQ5aijI0JOiTG+jttT8A8X6SMiUjsGVDkMdETILtGFup5RgJTsQoEqIiKqPRhQ5TCUlA4oANAR8cQUEZG6MaDKUUdHhOyXhviK94diPhERqR8DqhxGumKk5siQmlM0pFd86okX7xIRqV+t3/K9PFZ1xHiaK4Pjj8kAAMsXu+0WMKGIiNSOPahyiEqM5SVnF3WhCjiJj4hI7RhQFVjfvT5aSJU7mlxdgohI/RhQFejfxBCetgZK9xVwgT4iIrVjQFVCM/agiIhqHAOqEoY41lG6zR4UEZH6MaAqQSJWnizBHhQRkfoxoKqBC8gSEamfoAF14sQJDB48GC1atIBUKkVUVJSQ5VRaIYf4iIjUTtCAysrKgouLC+bOnQtDQ0MhS6kS//2piH+YK3QZRERvNEEDytvbG9OnT4efnx/EYs0ebfx9QEOl2zf/KRCoEiKi2kGzU0GDONXTVbot5oKxRERqpXVr8SUmJgr2+h/aijDhL33czxHjcUoKEkWaty/U634+mo7t025sn3ZTdfucnZ3LfVzrAqqiBpUnMTHxtV7vDCA6/Sm2/50NS0sLODvXrfax1OF126fp2D7txvZpNyHaxyG+Kioe2uNEPiIi9WJAVZHXi3X5cni1LhGRWgkaUJmZmbh48SIuXrwImUyGe/fu4eLFi7h7966QZZWrf2NDGEpEyOHVukREaiVoQJ0/fx7dunVDt27dkJ2djbCwMHTr1g3ffPONkGWVS18iwmdvGSGTAUVEpFaCTpLo2rUr0tPThSyhWurpiXEnk9dBERGpE89BVUM9PREy8tiDIiJSJwZUNdTTEyMjT4bbz9iLIiJSFwZUNdTXF2PvnRy0iU4RuhQiojcWA6oa2pjpVvwkIiJ6LQyoaqirK4aTSdH8kuFHnwpcDRHRm4kBVU11dYuWlNj2d7bAlRARvZkYUNX0ZTsToUsgInqjMaCqydPWAHv7mKNVfZ6PIiJSBwbUazA3ECMzX8Zlj4iI1IAB9Rr0JSIkPStE058eIj1XJnQ5RERvFAbUazCQFE2U+CdfjhDO5iMiUikG1GvQl/y77/tT9qCIiFSKAfUaDF4KqJf+SkREKsCAeg36kn//fvZJPpK4Nh8RkcowoF6DSCTC3f9YYXJbYwDA9XQGFBGRqgi6H9SbwFhXjImuJriaVoDMfJ6HIiJSFfagVMRYT4Rn+bweiohIVRhQKqIvFuG/J9N5PRQRkYowoFTkTlYhAGDXbS4eS0SkCgwoFVnWWYqv2psg4mqW0KUQEb0RGFAqYlFHgk9aGuFqej6GHk5FXiHPRxERvQ4GlArpSUSQiESIuZ2DOef+QRZn9RERVRsDSsUWd5ICAJZezsSeOzlc6ZyIqJoYUCo22KkORrvUBQCMPJaG7jGPIJfLIZMzqIiIqoIBpQbO9f7dxPB6RgH6/voE9dc/wD95HPIjIqosBpQaBDWrg5P9GyK2rzms60hwKiUPAGAf9RDjT6ULXB0RkXZgQKmBWCSCi6ku3C30sbqbKQCgp7U+AOBeViF+f5wnZHlERFqBAaVmnRrqYa5bPTQyLlr6PPZuDjx3P0ZGngwzf8/A0Qc5AldIRKSZuFismknEIoxyMUJmvgy+9oZ470AqAKBR1EMAwLeXMmGqL8I3HaVoaChGV0t9HHuYi6wCOU4k52K+u1TI8omIBCN4QIWHh2PZsmVISUlB8+bNERYWhs6dOwtdlsoZ6YrhaWuAcW8ZQSQCFl/MVDyWlivH6Pi0Ml938F4Ohjati5Et6qKuLju8RFR7CPqNt337dkyaNAn/+9//cOzYMXTs2BEDBw7E3bt3hSxLrb56ux6mt6+HtCBrnA+wwIouUuz0MUN9fTF6vDhP9bJbzwox8+w/sNn8EEFHnuJ5gQxyuRz3MgvQ5udkrPmzKOiKF6ndmZSNh88LkVcox9OcQhTI5Dhwr2gY8XmBDBNPp+N5AWcTEpHmE7QHtWLFCrz//vv48MMPAQALFizAoUOHEBkZia+++krI0tROJBKhiYkOmpgU/QhuvW8FoChgXM10oScRIfBAKv5Kz0e+DBjlUher/8zCjiTlxWgn/paB+Ie52H0nB0AdAE+VHm9rrovzT/Kx08cMRx7kYs1fWTj6IBfXMgowrZ0JdMXAw+eF+LBpXXTa8Qj/ca6Ddyz10bSeDtqZ60Ik4l72RCQMUXp6uiBXkObl5cHKygoRERHo37+/4v7x48fjzz//xN69e4Uoi4iINIRgQ3ypqakoLCxEgwYNlO5v0KABHj16JFBVRESkKQQ/615yCEkul3NYiYiIhAsoMzMzSCSSUr2lJ0+elOpVERFR7SNYQOnp6cHV1RVHjhxRuv/IkSNwc3MTqCoiItIUgs7i++STTxAaGor27dvDzc0NkZGRSE5ORnBwsJBlERGRBhD0HNSAAQMQFhaGBQsWoGvXrjh9+jS2bt0Ke3t7lb5PeHg4WrduDQsLC3h4eODkyZMqPb4qLF68GD169ICdnR0cHR0RGBiIP//8U+k5crkcYWFhaN68OSwtLeHr64u//vpL6Tnp6ekYOXIk7O3tYW9vj5EjRyI9XXmB2itXrqBv376wtLREixYtMG/ePMhreDuQRYsWQSqVYsKECYr7tL19ycnJGDVqFBwdHWFhYQE3NzccP378jWhfYWEhZs+erfj/qHXr1pg9ezYKCgq0sn0nTpzA4MGD0aJFC0ilUkRFRSk9XpNt2blzJ9zc3NCwYUO4ubkhJiZGre3Lz8/HV199hc6dO8Pa2hrNmjXDiBEjSl1/mpubiwkTJsDBwQHW1tYYPHgw7t+/r/Scu3fvIjAwENbW1nBwcMAXX3yBvDzltUaPHz8ODw8PWFhYoE2bNoiMjKx0OwSfJDFixAhcunQJjx49QlxcHN555x2VHl9bLgY+fvw4hg8fjn379mHXrl3Q0dFB//79kZb27woTS5cuxYoVKzBv3jwcPnwYDRo0gL+/P549e6Z4zogRI3Dx4kX8/PPPiI6OxsWLFxEaGqp4/J9//oG/vz8aNmyIw4cPY+7cufjuu++wfPnyGmvrmTNnsGHDBrRs2VLpfm1uX3p6Onx8fCCXy7F161b89ttvmD9/vtL5VG1u35IlSxAeHo558+YhISEBc+fOxdq1a7F48WKtbF9WVhZcXFwwd+5cGBoalnq8ptqSkJCAkJAQDBw4EPHx8Rg4cCCCgoLw+++/q619z58/xx9//IHx48cjLi4OP/zwA+7fv4/33ntP6ReOyZMnIyYmBhEREdi7dy+ePXuGwMBAFBYWAij6pSUwMBCZmZnYu3cvIiIisGvXLkydOlVxjKSkJAwaNAgdO3bEsWPH8Pnnn+OLL77Azp07K9UOwa6Dqim9evVCy5YtsWzZMsV97dq1g5+fn0ZfDJyZmQl7e3tERUWhT58+kMvlaN68OT766COMHz8eAJCdnQ1nZ2d8/fXXCA4OxrVr1+Dm5obY2Fi4u7sDAE6dOoU+ffrgzJkzcHZ2RkREBGbMmIHr168r/uEuWLAAkZGR+PPPP9U+gzIjIwMeHh5YunQp5s+fDxcXFyxYsEDr2zdr1iycOHEC+/btK/NxbW9fYGAgTE1NsXr1asV9o0aNQlpaGrZs2aLV7bOxscH8+fPxwQcfAKjZn1VwcDDS0tKwY8cORT1+fn4wNzdHRESEWtpXlqtXr8Ld3R0nTpxAy5YtkZGRAScnJ6xYsQKDBg0CANy7dw9vvfUWoqOj0atXLxw4cACDBg3CpUuXYGtrCwDYsmULPvvsMyQmJsLExARfffUVYmJicO7cOcV7jRkzBlevXsWBAwcqrF3wHpQ65eXl4cKFC+jZs6fS/T179sRvv/0mUFWVk5mZCZlMBqm0aLHY27dvIyUlRakthoaG6Ny5s6ItCQkJMDIyUppk4u7ujrp16yo9p1OnTkq/VfXq1QsPHz7E7du31d6usWPHws/PDx4eHkr3a3v79uzZg/bt2yM4OBhOTk7o0qULvv/+e8Vwjra3z93dHcePH8f169cBFH2hxcfHw8vL641o38tqsi1nzpwp9f3Uq1evGv9+Ku4ZFn/fXLhwAfn5+Uq12draolmzZkrta9asmSKcgKLac3NzceHCBcVzymrf+fPnkZ+fX2Fdb3RAafPFwJMmTcJbb72Fjh07AgBSUlIAoNy2PHr0CGZmZkq/ZYpEIpibmys9p6xjFD+mThs2bMCtW7eUhgCKaXv7kpKSEBERgcaNG2Pbtm0YNWoUZs6cibVr1wLQ/vaNHTsWgYGBcHNzg7m5Odzd3TFkyBCMGDECgPa372U12ZaUlBTBv5/y8vIwbdo09O7dGzY2Nor6JBIJzMzMXllbWe0refnQqz6DgoICpKamVlib4KuZ1wRtuxh4ypQpOH36NGJjYyGRSJQeq6gtZbWroucU/5avzs8kMTERs2bNwq+//go9Pb1XPk+DYXnXAAAVH0lEQVRb2yeTydC2bVvFsHGbNm1w69YthIeHY+TIkeXWpg3t2759O3766SeEh4ejefPmuHTpEiZNmgR7e3sMGzas3Nq0oX1lqam2CPn9VFBQgJEjRyIjIwM//vhjhc+vzGdQ8v7X+Xm+0T0obbwYePLkydi2bRt27dqFxo0bK+63sLAAUPq3yJfb0rBhQzx58kRplpBcLkdqaqrSc8o6BlD6N0ZVSkhIQGpqKjp16gQzMzOYmZnhxIkTCA8Ph5mZGerXr6/V7bOwsECzZs2U7mvatCnu3buneBzQ3vZNnz4dn376KQICAtCyZUsMHjwYn3zyCb799lsA2t++l9VkWywsLAT7fiooKMDw4cNx5coV7Ny5U/H/IFBUe2FhYaleTsnPoGTtJUetXvUZ6OjoKL3fq7zRAaVtFwNPnDgR0dHR2LVrF5o2bar0WKNGjWBhYaHUlpycHJw6dUrRlo4dOyIzMxMJCQmK5yQkJCArK0vpOadOnUJOzr87+R45cgRWVlZo1KiR2trm6+uLkydPIj4+XvGnbdu2CAgIQHx8PJycnLS6fe7u7rhx44bSfTdu3ICdnR0A7f/5PX/+vFRvXiKRQCaTvRHte1lNtqVDhw6CfD/l5+cjODgYV65cQUxMjCKUi7m6ukJXV1eptvv37ysmhwBF7bt27ZrS1PMjR45AX18frq6uiuccPXpU6dhHjhxB27ZtoaurW2GdkkmTJs2oZhu1grGxMcLCwmBpaQkDAwMsWLAAJ0+exPLly1GvXj2hy1MYP348fvrpJ6xfvx62trbIyspCVlYWgKKgFYlEKCwsxLfffgsnJycUFhZi6tSpSElJwZIlS6Cvrw9zc3P8/vvviI6ORuvWrXH//n2MGzcO7dq1U0x/dXR0xLp163Dp0iU4Ozvj1KlTmD59OsaOHavW/ykMDAzQoEEDpT8///wz7O3t8cEHH2h9+2xtbTFv3jyIxWJYWloiLi4Os2fPxrhx49C+fXutb9+1a9ewZcsWODk5QVdXF/Hx8fj6668xYMAA9OrVS+val5mZiatXryIlJQWbNm2Ci4sLTExMkJeXh3r16tVYW6ysrPDNN99AV1cXZmZm2LBhA6KiorB06VJYW1urpX1169bFhx9+iHPnzmHjxo0wNjZWfN9IJBLo6urCwMAAycnJWLt2LVq1aoWMjAyMGzcOJiYmmDlzJsRiMRo3boyYmBgcPnwYLVu2xNWrVzF+/HgMHDgQ/fr1AwA0adIES5YswePHj2FnZ4e9e/di0aJFmD17Npo3b15hO974aeZA0YW6S5cuRUpKClq0aIFvvvlG5ddbva7i2TMlTZw4EZMnTwZQNIQwd+5crF+/Hunp6Wjfvj0WLlwIFxcXxfPT0tIwceJE/PrrrwCAPn36YP78+UrHv3LlCsaPH49z585BKpUiODgYEydOrPExfl9fX8U08zehffv27cOsWbNw48YN2Nra4qOPPkJoaKjifbW5fc+ePcOcOXOwe/duPHnyBBYWFggICMAXX3wBAwMDrWtffHy84kv0ZUOGDMGqVatqtC07d+7E7NmzkZSUhCZNmmDatGl499131da+SZMmoU2bNmW+bsWKFYrp6Dk5Ofjyyy8RHR2NnJwcdOvWDYsWLVKatXf37l2MHz8ex44dg4GBAd577z3Mnj0b+vr/br56/PhxTJkyBVevXoWlpSXGjh2LkJCQSrWjVgQUERFpnzf6HBQREWkvBhQREWkkBhQREWkkBhQREWkkBhQREWkkBhQREWkkBhSpnaenJwICAqr12sjISEilUsUCnqRdAgIC8Omnnwpdxmt58OABGjZsiLi4OKFLqXUYULWMVCqt1J+SO4zWRjk5OaU+F3t7e/j5+ZVavoVKO3nyJI4ePYpx48ZV6XXx8fEICwtDZmammiqrGmtrawQGBmLOnDlCl1Lr8ELdWmbLli1Kt9evX4/ff/+91I6lbm5uSovVvo68vDyIRKJKrb1VUmFhIfLz8xWrFdSknJwcWFpaolevXhg0aBBkMhmSkpIQHh6O9PR07NixA127dq3xurRFYGAg8vPzsX379iq9LiwsDPPmzcO1a9dKrREnlPPnz6NHjx5KGxSS+tWK7TboX4GBgUq3jx49inPnzpW6/1UKCgogk8nK3TKjpKo8tySJRFJqkdKa5uzsrPT5+Pj4oGfPnli1atUrA0omkyEvL6/GgrU6Pxd1Sk5OxsGDB7F06VKhS1GJtm3bonHjxoiKimJA1SAO8dErXb9+HVKpFCtXrsSqVavQtm1bWFhY4I8//gAALF68GF5eXmjSpAksLCzwzjvv4Keffip1nJLnoF4+7saNG9G+fXtYWFiga9euOH78uNJryzoH5enpiS5duuDatWvo378/rKys0LRpU3zzzTdK2x8ARcv/h4aGwt7eHvb29ggJCcHdu3chlUoVW0VUVbt27WBkZISkpCQA/w4FTpo0CT///DM6deoECwsL7NmzB0BReCxcuBDt2rVDw4YN4eLigsmTJyt2MX3Z2rVr4erqCktLS3Tr1g2HDh1CSEgIOnToUObnV9bPRSaTYeXKlYo6HBwcEBoaiuTkZKX3SkxMxNChQ9G0aVNYWFigVatWCA4OVtoe4eDBg+jduzcaNWoEGxsbdOjQAZMmTarwM4qNjUVhYSG6d+9e6rFVq1bB3d0dVlZWaNy4MXr06IGNGzcCAGbMmIF58+YBAJo1a6YYWj1z5ozi9QcOHECfPn1gY2MDGxsb+Pn54ezZs0rvMWPGDEilUty8eRPBwcGws7ND48aN8fnnnysWYS529uxZDBgwAA4ODrCysoKrqytGjx6N3Nxcped1794de/bsKfVvjNSHPSiq0KZNm5CdnY0PP/wQBgYGMDc3BwAsX74c/fr1Q0BAAORyOXbt2oVRo0ZBLpdjyJAhFR5369atyMjIwLBhw6Crq4tVq1bh/fffx+XLl2FiYlLua9PS0uDv74//+7//w7vvvot9+/Zh/vz5aNKkieK9CwsLMXDgQFy4cAEhISFo3rw5Dh06hPfff/+1Po9Hjx4hMzOz1H42cXFxiI6OxogRI9CgQQM4ODgAAMaMGYMff/wR/fr1w+jRo3H58mWsXr0a58+fx549exQ9xNWrV2PSpElwd3fHqFGjkJKSguDgYMUupyW96ufy6aefYuvWrRgyZAhGjhyJBw8e4Pvvv8eZM2cQFxcHY2NjZGdnw9/fH3K5HKGhoWjQoIGi15OSkoKGDRvi4sWLGDJkCNq0aYPJkyfDwMAAf//9d6XOv506dQoNGjRQWlgUKArgyZMnIyAgAKGhocjLy8Nff/2F3377DcOGDcOAAQNw48YN7N69GwsWLFD8Oyj+LDdv3owxY8agZ8+e+PLLL5Gfn49NmzbB19cX+/btK7UI6tChQ2FnZ4fp06fjwoULiIyMxIMHDxS/SD18+BD+/v6wsrLCuHHjUK9ePdy5cwd79+5Fdna20qKn7dq1w/r163H9+vVSe3+RejCgqEL379/HuXPnFF+AxS5fvow6deoobo8aNQp9+/bFd999V6mAunv3Ls6ePatY/dnNzQ1eXl7YsWOH0i6tr6ppzZo1iqG34OBguLm5YePGjYr3/uWXX3Du3DmEhYVh9OjRAIARI0YgKCgIly5dqnT7c3NzkZqaqjgHVbxrrr+/v9Lzrl+/jpMnTyp9eZ0/fx4//vgjhg0bhmXLlinub9KkCWbMmIHo6GgEBgYiJycHYWFhaNeuHWJiYhTn69zd3REYGAhnZ+cyP4OSP5e4uDj88MMPiIiIUOq19u3bF7169cK6devw2Wef4cqVK7h37x5++ukn9O7dW/G8L774QvH3w4cPo6CgANu3b6/wF4aSEhMTy9y/ad++fXB1dUVERESZr2vdujVatmyJ3bt3491331U6B5WRkYHJkycjKChIqfcbFBQENzc3zJkzB1u3blU6XvGwXPHq4WZmZli2bBlOnDiBd955B6dOncI///yD2NhYpZXKp02bVqq24vZcu3aNAVVDOMRHFfLz8ysVTgAU4ZSfn4+0tDQ8ffoU3bp1w19//aW0SdurBAQEKG1N0KFDB+jr6+P27dsVvtbExASDBg1S3BaJROjcubNi2A0oGgrS19dHUFCQ0muL9+uprHXr1sHR0RHOzs7w8vLC5cuX8eWXX5baMqBLly6lvrhiY2MBAJ999pnS/SNHjoShoSH2798PAPjtt9+QkZGB4OBgpckkPj4+it5DSWX9XHbs2AGpVAoPDw+kpqYq/tjb28POzg7Hjh0DULRPGgAcOnQI2dnZZR7f2NgYcrkcv/76a5WHtVJTU8vcQsbY2Bh37txRDEdWxcGDB/Hs2TMMHDhQqW15eXno0qUL4uPjS9X58nYnxbeBon8bxfUART+ngoKCct/f1NQUAPD06dMq107Vwx4UVahJkyZl3r9z504sWrQIV65cQWFhodJjz549q3CCQPFusy+rV68e0tLSKqzJ1ta21P5AUqlU6bV3796FlZUVDA0NlZ7n5ORU4fFf1q9fPwwfPhwikQiWlpZo1KhRmW0r63O6c+cOdHR0Sj1Wp04d2NnZ4c6dO4pagaJN7kpydHRUCt7y3u/GjRtIT09/ZRuLQ6NZs2YICQnB2rVrsXnzZnTq1Ak+Pj4YNGiQ4os4MDAQUVFRCA0NxZQpU+Dh4QFfX1/4+flBR6fir46yQm3cuHE4efIkPDw84ODggJ49e6J///7o0qVLhccr3rG4b9++r3xOZmamInQAlAp3Gxsb1KlTR/F59+zZEz4+Ppg1axaWLFmCd955B3369MF7772nNDrwcntqet+02owBRRUq68s4Li4OQUFB6NKlC5YsWQJLS0vo6upiz549WLt2rWIr8PK8anZeZX5bF4vL7vxX5rVV7Q3Y2NiUebK/pPICuawvtcrW8arnlfV+MpkMlpaWWL16dZmvMTIyUvx98eLFCAkJQWxsLA4fPowpU6Zg4cKF+PXXX+Hk5AQjIyMcOHAA8fHxOHjwIA4dOoTt27dj1apV2LNnj9L5mZLMzMyQnp5e6v7WrVvj7Nmz2L9/Pw4fPozdu3cjPDwco0aNwty5c8v9HIr/TYWHh5fZowdQ6peRij53iUSCLVu2ICEhQVHTZ599hm+//RaHDh1SOs9Y3J6S5x5JfRhQVC07duyAsbExtm/frjQkVTx0ogns7Oxw9uxZZGdnK31x3bx5s8ZqsLe3R0FBAW7duqXUq8nOzsa9e/fQunVrRa3FtXXu3FnpGLdu3ar0VPsmTZogISEB7u7ulZri3qpVK7Rq1Qrjx4/HhQsX0LNnT6xZs0axy7FEIkH37t3RvXt3zJ49GytWrMDUqVMRGxsLPz+/Vx63adOmr/y3YGRkhAEDBmDAgAHIz8/H8OHDsXr1akyYMAFmZmav7KEU9xgbNGgADw+PCtsGFH2eL0/UuH//PrKzs0v13jt27IiOHTti2rRpiImJwdChQxEVFYUxY8YonlPci+X5p5rDc1BULcVfmC8P7T158qTMaeZC8fLyQm5uLtavX690/5o1a2qshuIJCCUvhF67di2ys7Ph7e0NoOgL0sTEBOvWrUN+fr7iefv27cOtW7cq/X4BAQHIz8/H/PnzSz0mk8kU508yMjJKDcs2b94cenp6yMjIAFD2uZbiWXLFz3kVd3d3pKamKobSipU8pq6urmJyQvEx69atCwClemC9e/eGkZER5s+fr/QZFXvy5Emp+0r+rItve3p6AkCZw8mvauO5c+dgampa5oQVUg/2oKhaevfujfDwcAQEBCAgIABPnz7FunXrYG1tjdTUVKHLAwD0798fy5cvx9SpU3Hz5k3FNPP79+8DqJlzCW3btsWQIUOwfv16pKWloWvXrrh8+TI2bNgAd3d3vPfeewCKhqYmTZqEKVOmoF+/fvD390dycjIiIyPRokWLUmHyKj169EBQUBAWL16MP/74Az169ICBgQGSkpIQExOD0NBQfPzxxzh48CCmT5+Od999F05OTigsLER0dDRycnIUsxO//vprnD9/Hp6enrC3t0dqaioiIiJgYmICLy+vcuvw8fGBRCLBkSNHlGZk9u3bF40aNULHjh3RoEED3LhxA2vXrkXbtm0V54tcXV0BANOnT4e/vz90dXXRo0cP1K9fH4sWLcLo0aPRtWtXBAQEoGHDhrh37x6OHTsGc3PzUkt0JSUlITAwEF5eXjh//jyioqLg7e2tOOe1fv16REVFwdfXF02aNEFWVhY2b94MXV1d9OvXT+lYR48eRd++fXkOqgYxoKhaPD09sWzZMixbtgyTJ0+Gra0tPvvsM+jq6uLzzz8XujwAgI6ODqKjozFlyhTF9ONevXrh+++/h7u7e7nnUFTpu+++g4ODA6KiorB3716Ym5sjNDQUU6dOVRq6+/jjjyEWi7Fy5UpMnz4dzZo1Q2RkJL7//ns8ePCg0u+3ZMkSxTU7c+bMgUQigY2NDXr37q3o0bm6uqJ79+6IjY1FcnIyDAwM0KJFC2zZsgU+Pj4AiiaHJCcnIyoqCqmpqTAzM4ObmxsmTpwIKyurcmuwsrKCp6cntm/frhRQw4cPx/bt27Fy5UpkZmbCysoKQUFBmDBhguI5Xbt2xcSJE7Fp0yYcOHAAMpkMBw4cQP369REYGAhbW1t8++23WL58OXJzc2FhYYEOHTrgww8/LFXHpk2b8PXXX2PmzJkQi8UICgrC7NmzFY97eHjg8uXL2LZtGx4/fgwTExO4urpiyZIlStdUnT9/Hrdv38aqVasq/XOg18e1+KjWSUhIgLe3NzZs2FDueRRN8fbbb8PR0bHUOoqa7vjx43j33XeRkJBQ5ZmTr2vGjBlYsmQJkpKSypzuXlVjxozB1atXNeoca23Ac1D0Rit5jY9cLsfKlSuho6NTajKC0Mq6diw2NhY3btxAt27dBKjo9XTp0gU9evSo9pJSmuLhw4fYsmVLmRfvknpxiI/eaP/9739RUFCADh06QCaTITY2FvHx8fj444/RoEEDoctTcvz4ccyYMQP9+vWDhYUFrly5gg0bNsDOzg5Dhw4Vurxq2bZtm9AlvDYrKyul9Qmp5jCg6I3WvXt3rFmzBgcPHkROTg4aNWqEWbNmaeQmeg4ODrCxsUFERATS0tJQr1499O/fH9OnT6/yUkNEbwKegyIiIo3Ec1BERKSRGFBERKSRGFBERKSRGFBERKSRGFBERKSRGFBERKSR/h82GZepuzgRdgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot losses\n",
    "\n",
    "with plt.style.context('fivethirtyeight'):\n",
    "    plt.plot(losses, linewidth = 1)\n",
    "    plt.xlabel('Steps (from step {})'.format(starting_step))\n",
    "    plt.ylabel('Losses')\n",
    "    plt.ylim((0, 1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "393\n",
      "INFO:tensorflow:Restoring parameters from ./checkpoints501/-12299\n",
      "1.\n",
      "--------------------------------\n",
      "तस्मात् ते गत्वा तस्य प्रयोजनीयद्रव्याणि संग्रहीतुं शोमिरोणीयानां ग्रामं प्रविविशुः। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0924\\u0938\\u094d\\u092e\\u093e\\u0924\\u094d \\u0924\\u0947 \\u0917\\u0924\\u094d\\u0935\\u093e \\u0924\\u0938\\u094d\\u092f \\u092a\\u094d\\u0930\\u092f\\u094b\\u091c\\u0928\\u0940\\u092f\\u0926\\u094d\\u0930\\u0935\\u094d\\u092f\\u093e\\u0923\\u093f \\u0938\\u0902\\u0917\\u094d\\u0930\\u0939\\u0940\\u0924\\u0941\\u0902 \\u0936\\u094b\\u092e\\u093f\\u0930\\u094b\\u0923\\u0940\\u092f\\u093e\\u0928\\u093e\\u0902 \\u0917\\u094d\\u0930\\u093e\\u092e\\u0902 \\u092a\\u094d\\u0930\\u0935\\u093f\\u0935\\u093f\\u0936\\u0941\\u0903\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and they reasoned for every multitude had prayed <ukn> and they could not his brother was healed <ukn> his blood <ukn> him <ukn> him <ukn> him <ukn> \n",
      "   Expected:  and sent messengers before his face <ukn> and they went <ukn> and entered into a village of the samaritans <ukn> to make ready for him <ukn> \n",
      "--------------------------------\n",
      "2.\n",
      "--------------------------------\n",
      "सर्वभूतस्थमात्मानं सर्वभूतानि चात्मनि <ukn> ईक्षते योगयुक्तात्मा सर्वत्र समदर्शन <ukn> <ukn> <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0938\\u0930\\u094d\\u0935\\u092d\\u0942\\u0924\\u0938\\u094d\\u0925\\u092e\\u093e\\u0924\\u094d\\u092e\\u093e\\u0928\\u0902 \\u0938\\u0930\\u094d\\u0935\\u092d\\u0942\\u0924\\u093e\\u0928\\u093f \\u091a\\u093e\\u0924\\u094d\\u092e\\u0928\\u093f <ukn> \\u0908\\u0915\\u094d\\u0937\\u0924\\u0947 \\u092f\\u094b\\u0917\\u092f\\u0941\\u0915\\u094d\\u0924\\u093e\\u0924\\u094d\\u092e\\u093e \\u0938\\u0930\\u094d\\u0935\\u0924\\u094d\\u0930 \\u0938\\u092e\\u0926\\u0930\\u094d\\u0936\\u0928 <ukn> <ukn> <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and pray over many days <ukn> that they should be received you with the house of many <ukn> many sent down of them <ukn> you <ukn> and them that should \n",
      "   Expected:  the true yogis <ukn> uniting their consciousness with god <ukn> see with equal eye <ukn> all living beings in god and god in all living beings <ukn> \n",
      "--------------------------------\n",
      "3.\n",
      "--------------------------------\n",
      "यतो <ukn> लोकोऽहं यञ्चाहं परिचरामि तदीय एको दूतो ह्यो रात्रौ ममान्तिके तिष्ठन् कथितवान् <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u092f\\u0924\\u094b <ukn> \\u0932\\u094b\\u0915\\u094b\\u093d\\u0939\\u0902 \\u092f\\u091e\\u094d\\u091a\\u093e\\u0939\\u0902 \\u092a\\u0930\\u093f\\u091a\\u0930\\u093e\\u092e\\u093f \\u0924\\u0926\\u0940\\u092f \\u090f\\u0915\\u094b \\u0926\\u0942\\u0924\\u094b \\u0939\\u094d\\u092f\\u094b \\u0930\\u093e\\u0924\\u094d\\u0930\\u094c \\u092e\\u092e\\u093e\\u0928\\u094d\\u0924\\u093f\\u0915\\u0947 \\u0924\\u093f\\u0937\\u094d\\u0920\\u0928\\u094d \\u0915\\u0925\\u093f\\u0924\\u0935\\u093e\\u0928\\u094d <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  then said the king of the jews <ukn> but afterward thou to live now as now <ukn> <ukn> <ukn> as i now <ukn> adultery the way of the \n",
      "   Expected:  for there stood by me this night the angel of god <ukn> whose i am <ukn> and whom i serve <ukn> \n",
      "--------------------------------\n",
      "4.\n",
      "--------------------------------\n",
      "हे प्रभो <ukn> मदीय एको दासः <ukn> भृशं व्यथितः <ukn> सतु शयनीय आस्ते। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0939\\u0947 \\u092a\\u094d\\u0930\\u092d\\u094b <ukn> \\u092e\\u0926\\u0940\\u092f \\u090f\\u0915\\u094b \\u0926\\u093e\\u0938\\u0903 <ukn> \\u092d\\u0943\\u0936\\u0902 \\u0935\\u094d\\u092f\\u0925\\u093f\\u0924\\u0903 <ukn> \\u0938\\u0924\\u0941 \\u0936\\u092f\\u0928\\u0940\\u092f \\u0906\\u0938\\u094d\\u0924\\u0947\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and he saith unto him <ukn> let it alone thou for the father <ukn> for it is it <ukn> for a sword <ukn> him <ukn> him <ukn> him \n",
      "   Expected:  and saying <ukn> lord <ukn> my servant lieth at home sick of the palsy <ukn> grievously tormented <ukn> \n",
      "--------------------------------\n",
      "5.\n",
      "--------------------------------\n",
      "यान् एतान् मनुष्यान् यूयमत्र समानयत ते <ukn> युष्माकं देव्या निन्दकाश्च न भवन्ति। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u092f\\u093e\\u0928\\u094d \\u090f\\u0924\\u093e\\u0928\\u094d \\u092e\\u0928\\u0941\\u0937\\u094d\\u092f\\u093e\\u0928\\u094d \\u092f\\u0942\\u092f\\u092e\\u0924\\u094d\\u0930 \\u0938\\u092e\\u093e\\u0928\\u092f\\u0924 \\u0924\\u0947 <ukn> \\u092f\\u0941\\u0937\\u094d\\u092e\\u093e\\u0915\\u0902 \\u0926\\u0947\\u0935\\u094d\\u092f\\u093e \\u0928\\u093f\\u0928\\u094d\\u0926\\u0915\\u093e\\u0936\\u094d\\u091a \\u0928 \\u092d\\u0935\\u0928\\u094d\\u0924\\u093f\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  that they have eaten with me <ukn> and they that have been written <ukn> then shall they be crucified your work <ukn> your peace <ukn> your peace <ukn> the \n",
      "   Expected:  for ye have brought hither these men <ukn> which are neither robbers of churches <ukn> nor yet blasphemers of your goddess <ukn> \n",
      "--------------------------------\n",
      "6.\n",
      "--------------------------------\n",
      "ततः सोवादीद् <ukn> यूयं न शृणुथ तर्हि कुतः पुनः श्रोतुम् इच्छथ <ukn> यूयमपि किं तस्य शिष्या भवितुम् इच्छथ <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0924\\u0924\\u0903 \\u0938\\u094b\\u0935\\u093e\\u0926\\u0940\\u0926\\u094d <ukn> \\u092f\\u0942\\u092f\\u0902 \\u0928 \\u0936\\u0943\\u0923\\u0941\\u0925 \\u0924\\u0930\\u094d\\u0939\\u093f \\u0915\\u0941\\u0924\\u0903 \\u092a\\u0941\\u0928\\u0903 \\u0936\\u094d\\u0930\\u094b\\u0924\\u0941\\u092e\\u094d \\u0907\\u091a\\u094d\\u091b\\u0925 <ukn> \\u092f\\u0942\\u092f\\u092e\\u092a\\u093f \\u0915\\u093f\\u0902 \\u0924\\u0938\\u094d\\u092f \\u0936\\u093f\\u0937\\u094d\\u092f\\u093e \\u092d\\u0935\\u093f\\u0924\\u0941\\u092e\\u094d \\u0907\\u091a\\u094d\\u091b\\u0925 <ukn> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and jesus saith unto them <ukn> receive ye not <ukn> ye will not put him <ukn> on your disciples <ukn> peter also <ukn> and christ <ukn> his disciples \n",
      "   Expected:  he answered them <ukn> i have told you already <ukn> and ye did not hear <ukn> wherefore would ye hear it again <ukn> will ye also be his disciples <ukn> \n",
      "--------------------------------\n",
      "7.\n",
      "--------------------------------\n",
      "<ukn> <ukn> पूर्व्वदिनस्य सायंकाल आगत <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "<ukn> <ukn> \\u092a\\u0942\\u0930\\u094d\\u0935\\u094d\\u0935\\u0926\\u093f\\u0928\\u0938\\u094d\\u092f \\u0938\\u093e\\u092f\\u0902\\u0915\\u093e\\u0932 \\u0906\\u0917\\u0924 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  for john was an oath at the jews <ukn> let us hold an melchisedec <ukn> as an oath is said unto him <ukn> an oath of what is without an unclean spirit \n",
      "   Expected:  and now when the even was come <ukn> because it was the preparation <ukn> that is <ukn> the day before the sabbath <ukn> \n",
      "--------------------------------\n",
      "8.\n",
      "--------------------------------\n",
      "यतः केचिदूचुर्योहन् श्मशानादुदतिष्ठत्। केचिदूचुः <ukn> एलियो दर्शनं दत्तवान् <ukn> एवमन्यलोका ऊचुः पूर्व्वीयः कश्चिद् भविष्यद्वादी समुत्थितः। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u092f\\u0924\\u0903 \\u0915\\u0947\\u091a\\u093f\\u0926\\u0942\\u091a\\u0941\\u0930\\u094d\\u092f\\u094b\\u0939\\u0928\\u094d \\u0936\\u094d\\u092e\\u0936\\u093e\\u0928\\u093e\\u0926\\u0941\\u0926\\u0924\\u093f\\u0937\\u094d\\u0920\\u0924\\u094d\\u0964 \\u0915\\u0947\\u091a\\u093f\\u0926\\u0942\\u091a\\u0941\\u0903 <ukn> \\u090f\\u0932\\u093f\\u092f\\u094b \\u0926\\u0930\\u094d\\u0936\\u0928\\u0902 \\u0926\\u0924\\u094d\\u0924\\u0935\\u093e\\u0928\\u094d <ukn> \\u090f\\u0935\\u092e\\u0928\\u094d\\u092f\\u0932\\u094b\\u0915\\u093e \\u090a\\u091a\\u0941\\u0903 \\u092a\\u0942\\u0930\\u094d\\u0935\\u094d\\u0935\\u0940\\u092f\\u0903 \\u0915\\u0936\\u094d\\u091a\\u093f\\u0926\\u094d \\u092d\\u0935\\u093f\\u0937\\u094d\\u092f\\u0926\\u094d\\u0935\\u093e\\u0926\\u0940 \\u0938\\u092e\\u0941\\u0924\\u094d\\u0925\\u093f\\u0924\\u0903\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and he put in a mountain <ukn> and in the place was put into the temple <ukn> the same place of the streets <ukn> so <ukn> so <ukn> \n",
      "   Expected:  and of some <ukn> that elias had appeared <ukn> and of others <ukn> that one of the old prophets was risen again <ukn> \n",
      "--------------------------------\n",
      "9.\n",
      "--------------------------------\n",
      "अननतरं योहनः शिष्यास्तद्वार्त्तां प्राप्यागत्य तस्य <ukn> श्मशानेऽस्थापयन्। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0905\\u0928\\u0928\\u0924\\u0930\\u0902 \\u092f\\u094b\\u0939\\u0928\\u0903 \\u0936\\u093f\\u0937\\u094d\\u092f\\u093e\\u0938\\u094d\\u0924\\u0926\\u094d\\u0935\\u093e\\u0930\\u094d\\u0924\\u094d\\u0924\\u093e\\u0902 \\u092a\\u094d\\u0930\\u093e\\u092a\\u094d\\u092f\\u093e\\u0917\\u0924\\u094d\\u092f \\u0924\\u0938\\u094d\\u092f <ukn> \\u0936\\u094d\\u092e\\u0936\\u093e\\u0928\\u0947\\u093d\\u0938\\u094d\\u0925\\u093e\\u092a\\u092f\\u0928\\u094d\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  and he was a great multitude <ukn> and touched all his son jesus <ukn> and he came and touched the people <ukn> the people and the chief priests prevailed <ukn> \n",
      "   Expected:  and when his disciples heard of it <ukn> they came and took up his corpse <ukn> and laid it in a tomb <ukn> \n",
      "--------------------------------\n",
      "10.\n",
      "--------------------------------\n",
      "तव पुत्रइति विख्यातो भवितुं न योग्योस्मि च <ukn> मां तव वैतनिकं दासं कृत्वा स्थापय। <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\\u0924\\u0935 \\u092a\\u0941\\u0924\\u094d\\u0930\\u0907\\u0924\\u093f \\u0935\\u093f\\u0916\\u094d\\u092f\\u093e\\u0924\\u094b \\u092d\\u0935\\u093f\\u0924\\u0941\\u0902 \\u0928 \\u092f\\u094b\\u0917\\u094d\\u092f\\u094b\\u0938\\u094d\\u092e\\u093f \\u091a <ukn> \\u092e\\u093e\\u0902 \\u0924\\u0935 \\u0935\\u0948\\u0924\\u0928\\u093f\\u0915\\u0902 \\u0926\\u093e\\u0938\\u0902 \\u0915\\u0943\\u0924\\u094d\\u0935\\u093e \\u0938\\u094d\\u0925\\u093e\\u092a\\u092f\\u0964 <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Translation:  therefore i go to thee <ukn> thy brother be not mine <ukn> and i am not the five hand <ukn> and thy brethren <ukn> not thy record <ukn> me <ukn> \n",
      "   Expected:  and am no more worthy to be called thy son <ukn> make me as one of thy hired servants <ukn> \n",
      "--------------------------------\n"
     ]
    }
   ],
   "source": [
    "f = pd.DataFrame(columns=['Translation', 'Expected', 'Bleu_Score'])\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    \n",
    "    # placeholders\n",
    "    encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "    decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "    # output projection\n",
    "    size = 512\n",
    "    w_t = tf.get_variable('proj_w', [en_vocab_size, size], tf.float32)\n",
    "    b = tf.get_variable('proj_b', [en_vocab_size], tf.float32)\n",
    "    w = tf.transpose(w_t)\n",
    "    output_projection = (w, b)\n",
    "    \n",
    "    # change the model so that output at time t can be fed as input at time t+1\n",
    "    outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                                encoder_inputs,\n",
    "                                                decoder_inputs,\n",
    "                                                tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                                num_encoder_symbols = sn_vocab_size,\n",
    "                                                num_decoder_symbols = en_vocab_size,\n",
    "                                                embedding_size = 100,\n",
    "                                                feed_previous = True, # <-----this is changed----->\n",
    "                                                output_projection = output_projection,\n",
    "                                                dtype = tf.float32)\n",
    "    \n",
    "    # ops for projecting outputs\n",
    "    outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "    sn_sentences_encoded = X_train\n",
    "    en_sentences_encoded = Y_train\n",
    "    sn_sentences_extra = []#'तर्हि मम वाक्यानि कथं प्रत्येष्यथ', 'मानव इव कोपि कदापि नोपादिशत्']\n",
    "    en_sentences_extra = []#'how shall ye believe my words', 'Never man spake like this man']\n",
    "    sn_sentences_encoded += [[sn_word2idx[word] if word in sn_word2idx else sn_word2idx['<ukn>'] for word in sentence.encode('unicode-escape').decode('utf-8').split()] for sentence in sn_sentences_extra]\n",
    "    en_sentences_encoded += [[en_word2idx[word] if word in en_word2idx else en_word2idx['<ukn>'] for word in sentence.split()] for sentence in en_sentences_extra]\n",
    "    # padding to fit encoder input\n",
    "    for i in range(len(sn_sentences_encoded)):\n",
    "        sn_sentences_encoded[i] += (15 - len(sn_sentences_encoded[i])) * [sn_word2idx['<pad>']]\n",
    "    for i in range(len(en_sentences_encoded)):\n",
    "        en_sentences_encoded[i] += (15 - len(en_sentences_encoded[i])) * [en_word2idx['<pad>']]\n",
    "    \n",
    "    # restore all variables - use the last checkpoint saved\n",
    "    saver = tf.train.Saver()\n",
    "    path = tf.train.latest_checkpoint(checkpointsPath)\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        # restore\n",
    "        saver.restore(sess, path)\n",
    "        \n",
    "        # feed data into placeholders\n",
    "        feed = {}\n",
    "        for i in range(input_seq_len):\n",
    "            feed[encoder_inputs[i].name] = np.array([sn_sentences_encoded[j][i] for j in range(len(sn_sentences_encoded))], dtype = np.int32)\n",
    "            \n",
    "        feed[decoder_inputs[0].name] = np.array([en_word2idx['<go>']] * len(sn_sentences_encoded), dtype = np.int32)\n",
    "        \n",
    "        # translate\n",
    "        output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "        \n",
    "        # decode seq.\n",
    "        for i in range(len(sn_sentences_encoded)):\n",
    "            print('{}.\\n--------------------------------'.format(i+1))\n",
    "            ouput_seq = [output_sequences[j][i] for j in range(output_seq_len)]\n",
    "            #decode output sequence\n",
    "            words = decode_output(ouput_seq)\n",
    "            expected = [en_idx2word[word] for word in en_sentences_encoded[i]]\n",
    "        \n",
    "            print(\" \". join([convert_sanskrit(sn_idx2word[word]) for word in sn_sentences_encoded[i] if word is not sn_word2idx['<pad>']]))\n",
    "            #print(\" \". join([(en_idx2word[word]) for word in en_sentences_encoded[i]]))\n",
    "            \n",
    "            translation_bleu = []\n",
    "            print('Translated: ', end = \" \")\n",
    "            for j in range(len(words)):\n",
    "                if words[j] not in ['<eos>', '<pad>', '<go>']:\n",
    "                    print((words[j]), end = \" \")\n",
    "                    translation_bleu.append(words[j])\n",
    "            print()\n",
    "            expected_bleu = []\n",
    "            print('  Expected: ', end = \" \")\n",
    "            for j in range(len(expected)):\n",
    "                if expected[j] not in ['<eos>', '<pad>', '<go>']:\n",
    "                    print(expected[j], end = \" \")\n",
    "                    expected_bleu.append(expected[j])\n",
    "            print()\n",
    "            \n",
    "            bleu_score = sentence_bleu([expected_bleu], translation_bleu)\n",
    "            \n",
    "            df = df.append(other = {'Translation':translation_bleu, 'Expected':expected_bleu, 'Bleu_Score':bleu_score},ignore_index=True)\n",
    "            print('Bleu Score: ', end = \" \")\n",
    "            print(bleu_score)\n",
    "\n",
    "            print('\\n--------------------------------')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}