chatbot_reddit.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "chatbot_reddit.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "oAP57D6jx-Wo",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "5a829bb8-b534-4efb-d683-0aabe0ea70f0",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903852587,
          "user_tz": -330,
          "elapsed": 15121,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()\n",
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\r\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "T8Nsg92RKL1m",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# !kill -9 -1 "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AYvUi_rpyRYI",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "aa4b77d9-ba68-4fe9-e2ec-71a246150cbc",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903891370,
          "user_tz": -330,
          "elapsed": 4084,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!mkdir -p drive\n",
        "!google-drive-ocamlfuse drive\n",
        "!free -g"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              total        used        free      shared  buff/cache   available\r\n",
            "Mem:             12           0           1           0          10          11\r\n",
            "Swap:             0           0           0\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "cNC33BGgyVN4",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ab467436-ad7c-4c25-afed-c8639f542c04",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903893780,
          "user_tz": -330,
          "elapsed": 1843,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "datalab  drive\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "44Ma7a_PQOOD",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.text import one_hot\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from scipy.ndimage.interpolation import shift\n",
        "from keras.models import load_model\n",
        "import keras\n",
        "import os\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "X-Jr60EUQOOG",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "nb_samples = 25000\n",
        "nb_words  = 1000"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_248aCPvQOOJ",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def return_text(file):\n",
        "    with open(file, 'r') as f:\n",
        "        lines = f.readlines()\n",
        "    return lines"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ioxsyuC0QOOM",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# test_from = return_text('/data/test.from')\n",
        "train_from = return_text('drive/projects/chatbot-reddit/data/train.from')\n",
        "# test_to = return_text('drive/projects/chatbot-reddit/data/test.to')\n",
        "train_to = return_text('drive/projects/chatbot-reddit/data/train.to')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_OMBNJ3gx8G1",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def clean_sentences(sentences):\n",
        "    n_sentences=[]\n",
        "    temp = ''\n",
        "    for sentence in sentences:\n",
        "        if '\\n' in sentence:\n",
        "            loc = sentence.find('\\n')\n",
        "        temp = \"sos \"+sentence[:loc] + \" eos\"\n",
        "        n_sentences.append(temp)\n",
        "        temp=''\n",
        "    return n_sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UBxKM_frx8G4",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "cleaned_trainfrom = clean_sentences(train_from)\n",
        "cleaned_trainto = clean_sentences(train_to)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pkMQIz1om_uy",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "b92711e3-c615-4fd0-edae-6e3bc208ce03",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903911727,
          "user_tz": -330,
          "elapsed": 1005,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "cleaned_trainfrom[0]"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'sos As always Dilbert is so true. newlinechar  newlinechar Funny thing about Strategy is that you can have the right Strategy and still find yourself worse off. eos'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "metadata": {
        "id": "v6DWSFicQOOP",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "tokenizer = Tokenizer(num_words = nb_words,lower=True)\n",
        "tokenizer.fit_on_texts(cleaned_trainfrom + cleaned_trainto)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "STfEY2CmtMB0",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a4fb0ff2-ed2c-4dfb-8de7-2840b4f85e7d",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903917927,
          "user_tz": -330,
          "elapsed": 1102,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "len(tokenizer.word_index)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "48311"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "metadata": {
        "id": "tYXUiFK4QOOS",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "train_from_sequences = tokenizer.texts_to_sequences(cleaned_trainfrom)\n",
        "train_to_sequences = tokenizer.texts_to_sequences(cleaned_trainto)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hfZBW2wZQOOU",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# test_to_sequences = tokenizer.texts_to_sequences(test_to)\n",
        "# test_from_sequences = tokenizer.texts_to_sequences(test_from)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lxsF4W9vQOOX",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "max_seq_train_from = max([len(seq) for seq in train_from_sequences])\n",
        "max_seq_train_to = max([len(seq) for seq in train_to_sequences])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1pPmHltHQOOa",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "29ac673c-7b23-432a-98dc-f0b4ecf83dd3",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903924859,
          "user_tz": -330,
          "elapsed": 1156,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "max_seq_train_from,max_seq_train_to"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(87, 77)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "OlFPYWhjQOOf",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "data_trainfrom = pad_sequences(train_from_sequences, maxlen=max_seq_train_from)\n",
        "data_trainto = pad_sequences(train_to_sequences, maxlen=max_seq_train_to)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "apfAb5qUQOOj",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# data_testfrom = pad_sequences(train_from_sequences, maxlen=max_seq_train_from)\n",
        "# data_testto = pad_sequences(train_to_sequences, maxlen=max_seq_train_to)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x8w6qyQ1QOOm",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "d37f976e-b90e-404d-a131-236722f3edc3",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903929786,
          "user_tz": -330,
          "elapsed": 1228,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "data_trainfrom[0]"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   1,  31, 179,\n",
              "        13,  36, 214,   4,   4, 285, 132,  48,  13,  12,  10,  49,  20,\n",
              "         3,  92,   9, 101, 192, 430, 515, 129,   2], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "metadata": {
        "id": "n-oxh0vZQOO5",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# data_trainfrom = data_trainfrom[:nb_samples]\n",
        "# data_trainto = data_trainto [:nb_samples]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cnaI1PbMLw9X",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c55f56d9-9464-4541-8cf0-49b6308df6ec",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903936394,
          "user_tz": -330,
          "elapsed": 1137,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "data_trainfrom.shape,data_trainto.shape"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((28778, 87), (28778, 77))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "metadata": {
        "id": "y2WSZQnVOQ1j",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# def shift2(arr,num):\n",
        "#     arr=np.roll(arr,num)\n",
        "#     if num<0:\n",
        "#          np.put(arr,range(len(arr)+num,len(arr)),0)\n",
        "#     elif num > 0:\n",
        "#          np.put(arr,range(num),0)\n",
        "#     return arr"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Of_vZ-P7OTDw",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# target_data = shift2(data_trainto, -1)\n",
        "# target_data = target_data[:nb_samples]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yyRBm3MaZIjd",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "new_tokens = {}\n",
        "for token, val in  tokenizer.word_index.items():\n",
        "  if(val<nb_words):\n",
        "    if re.match('^[a-zA-Z0-9]',token ):\n",
        "        new_tokens[token] = val\n",
        "  else:\n",
        "    break\n",
        "num_tokens = len(new_tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "A2qQPSl0QOPC",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "index2word = {}\n",
        "for word,i in new_tokens.items():\n",
        "    index2word[i] = word"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yDXt3LPUOmuR",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6c4aac92-3f5c-4f00-ab8e-041eae3ed228",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903959067,
          "user_tz": -330,
          "elapsed": 1182,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "index2word[998]"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'million'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "metadata": {
        "id": "yb8wdo3DSKCR",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "88be1162-ac0f-4572-d43f-6ba39ca632c7",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903962863,
          "user_tz": -330,
          "elapsed": 997,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "new_tokens['eos']"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "metadata": {
        "id": "-0a0zFVTQVH7",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# X = np.zeros((nb_samples,max_seq_train_from, num_tokens+1), dtype='float32')\n",
        "# X_context = np.zeros((nb_samples,max_seq_train_to, num_tokens+1), dtype='float32')\n",
        "nb_samples = data_trainfrom.shape[0]\n",
        "Y = np.zeros((nb_samples,max_seq_train_to, nb_words), dtype='float32')\n",
        "\n",
        "\n",
        "for i, target_text in enumerate(cleaned_trainto):\n",
        "  for t, word in enumerate(target_text.split(' ')):\n",
        "    if word in new_tokens and i<nb_samples and new_tokens[word]< nb_words:\n",
        "      if t > 0:          \n",
        "        # decoder_target_data will be ahead by one timestep\n",
        "        # and will not include the start character.\n",
        "        Y[i, t - 1, new_tokens[word]] = 1."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZU1Bebh8weUr",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ccea183b-0c26-4e81-8b22-e26352edc5b0",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531903973166,
          "user_tz": -330,
          "elapsed": 1156,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "Y.shape"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(28778, 77, 1000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "metadata": {
        "id": "lkN2e-8sgYIR",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "path = 'drive/projects/chatbot-reddit'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3p6zqAWD_ZYn",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.layers import Input, Embedding, LSTM, Dense \n",
        "from keras.models import Model\n",
        "from keras import optimizers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "farGERC6GMgF",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "latent_dim = 256\n",
        "batch_size = 128\n",
        "epochs = 50"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "s6_pkXk0GLQk",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Define an input sequence and process it.\n",
        "encoder_inputs = Input(shape=(None,))\n",
        "x = Embedding(nb_words, latent_dim)(encoder_inputs)\n",
        "encoder, state_h, state_c = LSTM(latent_dim,\n",
        "                           return_state=True)(x)\n",
        "encoder_states = [state_h, state_c]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CQbMgaf_GO3Z",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "y = Embedding(nb_words, latent_dim)(decoder_inputs)\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True,return_state=True)\n",
        "d_out, _, _ = decoder_lstm(y, initial_state=encoder_states)\n",
        "decoder_dense = Dense(nb_words, activation='softmax')\n",
        "decoder_outputs = decoder_dense(d_out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hF9mIWgsGT9a",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Define the model that will turn\n",
        "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "# Compile & run training\n",
        "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GAGCTl7GGbnj",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "callbacks_list = [\n",
        "        keras.callbacks.ReduceLROnPlateau(\n",
        "            monitor='loss',\n",
        "            factor=0.1,\n",
        "            patience=10,\n",
        "        ),\n",
        "        keras.callbacks.ModelCheckpoint(\n",
        "            filepath=os.path.join(path,'models/chatbotC.h5'),\n",
        "            monitor='loss',\n",
        "            save_best_only=True,\n",
        "        ),\n",
        "]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Zl0iAhMXgKx1",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 1788
        },
        "outputId": "88964cb8-a409-48cc-831b-7f48b7ff0a89",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531909789180,
          "user_tz": -330,
          "elapsed": 4527463,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Note that `decoder_target_data` needs to be one-hot encoded,\n",
        "# rather than sequences of integers like `decoder_input_data`!\n",
        "model.load_weights(os.path.join(path,'models/chatbotC.h5'))\n",
        "history = model.fit([data_trainfrom, data_trainto], Y,\n",
        "          batch_size=batch_size,\n",
        "          epochs=epochs,\n",
        "          validation_split=0.1,\n",
        "          callbacks = callbacks_list)"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 25900 samples, validate on 2878 samples\n",
            "Epoch 1/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6203 - val_loss: 0.6230\n",
            "Epoch 2/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6193 - val_loss: 0.6239\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/keras/engine/topology.py:2379: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_1/while/Exit_2:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_1/while/Exit_3:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
            "  str(node.arguments) + '. They will not be included '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 3/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6183 - val_loss: 0.6237\n",
            "Epoch 4/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6172 - val_loss: 0.6236\n",
            "Epoch 5/50\n",
            "19840/25900 [=====================>........] - ETA: 19s - loss: 0.6162"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6161 - val_loss: 0.6248\n",
            "Epoch 6/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6147 - val_loss: 0.6253\n",
            "Epoch 7/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6132 - val_loss: 0.6260\n",
            "Epoch 8/50\n",
            " 1024/25900 [>.............................] - ETA: 1:24 - loss: 0.5862"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6115 - val_loss: 0.6264\n",
            "Epoch 9/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6096 - val_loss: 0.6272\n",
            "Epoch 10/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.6084"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6074 - val_loss: 0.6285\n",
            "Epoch 11/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6050 - val_loss: 0.6296\n",
            "Epoch 12/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.6026 - val_loss: 0.6306\n",
            "Epoch 13/50\n",
            " 1408/25900 [>.............................] - ETA: 1:21 - loss: 0.5748"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5998 - val_loss: 0.6320\n",
            "Epoch 14/50\n",
            "25900/25900 [==============================] - 86s 3ms/step - loss: 0.5968 - val_loss: 0.6337\n",
            "Epoch 15/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.5940"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5936 - val_loss: 0.6359\n",
            "Epoch 16/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5903 - val_loss: 0.6380\n",
            "Epoch 17/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5868 - val_loss: 0.6403\n",
            "Epoch 18/50\n",
            " 1408/25900 [>.............................] - ETA: 1:22 - loss: 0.5775"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5832 - val_loss: 0.6428\n",
            "Epoch 19/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5795 - val_loss: 0.6433\n",
            "Epoch 20/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.5752"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5756 - val_loss: 0.6458\n",
            "Epoch 21/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5716 - val_loss: 0.6484\n",
            "Epoch 22/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5678 - val_loss: 0.6518\n",
            "Epoch 23/50\n",
            " 1408/25900 [>.............................] - ETA: 1:22 - loss: 0.5730"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5638 - val_loss: 0.6540\n",
            "Epoch 24/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5598 - val_loss: 0.6574\n",
            "Epoch 25/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.5542"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5558 - val_loss: 0.6597\n",
            "Epoch 26/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5518 - val_loss: 0.6630\n",
            "Epoch 27/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5478 - val_loss: 0.6659\n",
            "Epoch 28/50\n",
            " 1408/25900 [>.............................] - ETA: 1:22 - loss: 0.5376"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5436 - val_loss: 0.6694\n",
            "Epoch 29/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5398 - val_loss: 0.6705\n",
            "Epoch 30/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.5327"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5359 - val_loss: 0.6762\n",
            "Epoch 31/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5319 - val_loss: 0.6782\n",
            "Epoch 32/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5281 - val_loss: 0.6842\n",
            "Epoch 33/50\n",
            " 1408/25900 [>.............................] - ETA: 1:22 - loss: 0.5240"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5243 - val_loss: 0.6857\n",
            "Epoch 34/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5205 - val_loss: 0.6886\n",
            "Epoch 35/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.5153"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5169 - val_loss: 0.6916\n",
            "Epoch 36/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5132 - val_loss: 0.6944\n",
            "Epoch 37/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5096 - val_loss: 0.6988\n",
            "Epoch 38/50\n",
            " 1408/25900 [>.............................] - ETA: 1:21 - loss: 0.5114"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5061 - val_loss: 0.7024\n",
            "Epoch 39/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.5026 - val_loss: 0.7078\n",
            "Epoch 40/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.4989"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4994 - val_loss: 0.7103\n",
            "Epoch 41/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4960 - val_loss: 0.7156\n",
            "Epoch 42/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4925 - val_loss: 0.7168\n",
            "Epoch 43/50\n",
            " 1408/25900 [>.............................] - ETA: 1:22 - loss: 0.4714"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4893 - val_loss: 0.7222\n",
            "Epoch 44/50\n",
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4861 - val_loss: 0.7224\n",
            "Epoch 45/50\n",
            "22784/25900 [=========================>....] - ETA: 10s - loss: 0.4808"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 87s 3ms/step - loss: 0.4831 - val_loss: 0.7252\n",
            "Epoch 46/50\n",
            "25900/25900 [==============================] - 85s 3ms/step - loss: 0.4799 - val_loss: 0.7316\n",
            "Epoch 47/50\n",
            "25900/25900 [==============================] - 86s 3ms/step - loss: 0.4770 - val_loss: 0.7333\n",
            "Epoch 48/50\n",
            " 1408/25900 [>.............................] - ETA: 1:20 - loss: 0.4808"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 85s 3ms/step - loss: 0.4741 - val_loss: 0.7374\n",
            "Epoch 49/50\n",
            "25900/25900 [==============================] - 85s 3ms/step - loss: 0.4709 - val_loss: 0.7427\n",
            "Epoch 50/50\n",
            "22784/25900 [=========================>....] - ETA: 9s - loss: 0.4678 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "25900/25900 [==============================] - 85s 3ms/step - loss: 0.4682 - val_loss: 0.7452\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GOZKV1cGIOb_",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def sample(preds, temperature= 0.2):\n",
        "    preds = np.reshape(preds,preds.shape[-1])\n",
        "    preds = np.asarray(preds).astype('float64')\n",
        "    preds = np.log(preds)/ temperature\n",
        "    exp_preds = np.exp(preds)\n",
        "    preds = exp_preds / np.sum(exp_preds)\n",
        "    probas = np.random.multinomial(1,preds, 1)\n",
        "    return np.argmax(probas)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7G6U4DSjbH8x",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 629
        },
        "outputId": "77116e73-e8ab-49b3-ba33-71c2f9f04747",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531909821726,
          "user_tz": -330,
          "elapsed": 1133,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Define sampling models\n",
        "encoder_model = Model(encoder_inputs, encoder_states)\n",
        "print(encoder_model.summary())\n",
        "                    \n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "y = Embedding(nb_words, latent_dim)(decoder_inputs)\n",
        "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
        "decoder_outputs, state_h, state_c = decoder_lstm(y, initial_state=decoder_states_inputs)\n",
        "decoder_states  = [state_h, state_c]\n",
        "\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "decoder_model   = Model([decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states)\n",
        "print(decoder_model.summary())\n"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         (None, None)              0         \n",
            "_________________________________________________________________\n",
            "embedding_1 (Embedding)      (None, None, 256)         256000    \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                [(None, 256), (None, 256) 525312    \n",
            "=================================================================\n",
            "Total params: 781,312\n",
            "Trainable params: 781,312\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_11 (InputLayer)           (None, None)         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_5 (Embedding)         (None, None, 256)    256000      input_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "input_9 (InputLayer)            (None, 256)          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_10 (InputLayer)           (None, 256)          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   [(None, None, 256),  525312      embedding_5[0][0]                \n",
            "                                                                 input_9[0][0]                    \n",
            "                                                                 input_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, None, 1000)   257000      lstm_2[3][0]                     \n",
            "==================================================================================================\n",
            "Total params: 1,038,312\n",
            "Trainable params: 1,038,312\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Mb46tLwnbt_w",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bc2d1487-b9ab-43e4-f06a-18ace9b7d4ba",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531909824349,
          "user_tz": -330,
          "elapsed": 909,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "index2word[1]"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'sos'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 77
        }
      ]
    },
    {
      "metadata": {
        "id": "OLKvygYeWoOL",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "ef084159-ea5f-4882-a2c2-29c94048bdd5",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531909826010,
          "user_tz": -330,
          "elapsed": 646,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "data_trainto[2]"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "         0,   0,   0,   1, 113,   7,   3, 304,   8,  59,  96,  10,  20,\n",
              "        31,   5,   7,  34, 758,   7,   5, 413, 602,  14, 584,   7,  88,\n",
              "         5, 413,   7,   3,   8,   5,   9,   7,  31,   5, 301,   2],\n",
              "      dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "metadata": {
        "id": "F8AhDxOUbHwg",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    states_value = encoder_model.predict(input_seq)\n",
        "\n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,max_seq_train_to))\n",
        "    # Populate the first character of target sequence with the start character.\n",
        "    target_seq[0,-1] = new_tokens['sos']\n",
        "\n",
        "    # Sampling loop for a batch of sequences\n",
        "    # (to simplify, here we assume a batch of size 1).\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "      output_tokens, h, c = decoder_model.predict([input_seq] + states_value)\n",
        "      # Sample a token\n",
        "      sampled_token_index = sample(output_tokens[0, -1, :] ,0.9)\n",
        "      sampled_char = index2word[sampled_token_index]\n",
        "      decoded_sentence += sampled_char +\" \"\n",
        "\n",
        "      # Exit condition: either hit max length\n",
        "      # or find stop character.\n",
        "      if (sampled_char == 'eos' or\n",
        "         len(decoded_sentence) > max_seq_train_to):\n",
        "          stop_condition = True\n",
        "\n",
        "      # Update the target sequence (of length 1).\n",
        "      target_seq = np.zeros((1,max_seq_train_to))\n",
        "      target_seq[0, -1] = sampled_token_index\n",
        "\n",
        "      # Update states\n",
        "      states_value = [h, c]\n",
        "\n",
        "    return decoded_sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Qvlr_YyaQOP9",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def make_inference(sentence):\n",
        "    input_text_seq = tokenizer.texts_to_sequences(sentence)\n",
        "    input_data_padded = pad_sequences(input_text_seq, maxlen=max_seq_train_from)\n",
        "    decoded_sentence = decode_sequence(input_data_padded)\n",
        "    print('Question:', sentence)\n",
        "    print('Reply:', decoded_sentence)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I13eKsBgrCD4",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 280
        },
        "outputId": "1ea6406d-4da9-4256-a195-cdf7331b7c02",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1531909913344,
          "user_tz": -330,
          "elapsed": 1342,
          "user": {
            "displayName": "Arth Dharaskar",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "112750200661093479151"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(1, len(loss) + 1)\n",
        "plt.figure()\n",
        "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEHCAYAAABV4gY/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8VFX6+PHPlBRC0QihBAEVyUGw\nrLgoiAgasFDUuDbWsiwiKrLC7rqK+xUVC1YUVPwadV07oq6ofGWtKD8ECwIqaHjoNYARgQRCypTf\nH3cmpMwkkzKZ9rxfr7wyc+bemXNmkidPnnvuuTav14tSSqn4Yo90B5RSSjU9De5KKRWHNLgrpVQc\n0uCulFJxSIO7UkrFIQ3uSikVh5yR7oCKLGPM/wJn+e52B/KBg777fUWkqB7PtRoYJCK7atnmAWCz\niDzTwC43OWPMp8CrIvJiEzyXF+gC9AVGisiYhr6eMeY6EXnOd7vO97YefXwRWCci9zX2uVT00uCe\n4ETkRv9tY8wm4CoR+bKBz9UzhG1ub8hzxxoRmQvMbej+xpiOwK3Ac77nq/O9VaoyDe6qVsaYL4DF\nwMXAtcB64CXgKCAFeFJEHvNt689ajwUeAL4ALgJSgdEisrBy1uj7Y/KA73m7AK+LyN99z/VPYBKw\nGfg3cKuIHBWgf2OBv2P9LO8ArhaRzcaY0cBwoBAYCLiAS0XkJ2PMMcBsoB3wNQF+D4wxw4CHROSE\nSm3fA5OBFcHeg0rbjsb6QzmkttczxlwA3A8kA/uBa0Xke2AJcKQvYz8RKAW6iMg2Y8zNwA1YZVUB\nxopIge+93QycDmQBa4ALRaS4+vgqvf6JwP8CbYES4DYR+cgY0wp4BejpG+NnwHjf7RrtIlIe7DVU\nZGjNXYXiFKC3iCwB7gA2+jLJbOABY0yXAPucDHwtIscBT/v2C+RMoL/vNf5ijDnSGNMbK2s9CSsw\nXxZoR2NMe+ApYKiI9ADWAVMqbTIMeFpEsoDPsf5YADwIfCYi3YGZwIAAT/8pVnA92vdaRwNH+tpD\nfQ/8Ar6eMcaJ9UfiOhExwHvAo759xgBbRKSniJRVGnM/4B/AYN/rb8H6A+l3KXA5VoktA8gJ1ilj\njB14A3jK91xjgdnGmNbAn4C9vs8vC+uPY+9a2lWU0eCuQjFfRDy+2zcDfwEQkQ3ATuDoAPsUich7\nvtvLga5Bnvt1EXGLSD6wCyuDPxP4QkR2iEgJ8EKgHUXkF6CNiGzzNS0Cjqm0yc8isixAH84E5vie\n41tgdYDnLgPmARf4mnKAd0XEVY/3wC/g6/meq72IfB2k/4EMB972jR3geeCcSo9/ICK/+Z57JcHf\nd3x97ogV4BGR77Ay/77AL0B/Y8w5gENEbvT9RxGsXUUZLcuoUPxW6XZfrEy1K+AGOhE4SdhX6bYb\ncAR57kDbpVd7ze2BdjTGOIB7fKUNB9AaqxRRVx+OqPbYniB9exuYiJVtXwTc62sP9T3wq+31bjbG\n/AmrxJEK1LXYUwbWQe/Kz9W+0v1Q33f/c+0VkcqvuQfrD84bxpgjsMbc0xjzKvA3EXkrSHtpHf1W\nzUwzd1Vfr2IFvSzfv/IFYXiNQqBVpfudgmx3OVZmfaavrHFXiM+/Bzis0v2MINt9BPzOGNMDqwSx\nwNde3/cg4OsZY04HbgMu8PV/bAh934VVH/dr62triF3AEcYYW6DnE5FcETkN6IVVNrumtnYVXTS4\nq/pqDywTEa8v42xJ1UDcFL4FzjLGtDPGpGDVeYP1ZZOI/GqMaYtVmw+lL1/hq0X7AuyxgTbyZaMf\nAQ8D74mIu9Lr1uc9CPZ67bHKHFuMMWm+cbb0BdtyoJWvLl/ZB8DFvvECXO9ra4hNwDasP5L+vnUE\nvjXGTDHGjAEQke3ARsAbrL2Br6/CSIO7qq8pwFxjzI9YAS0XeM4Y072pXsBXl34Ja1bKAqzad6AA\nMhtoa4xZ57t9B9DFGDO9jpe4FRhpjFkPTAA+qWXbt7FKMm9WaqvvexDs9T7EKrGsBz4GZmCVVd4G\nfsQqTe30lX+AivfmQWCRbybN4cD/1DHegHzlmCuACcaYPOAJrBlFB7BmxFxtjBHf65T52oK1qyhj\n0/XcVTQyxtj8tWBjzHDgPhE5OcLdUipm6AFVFXWMMRnAamNMH6ypfpdhlTaUUiHSsoyKOiJSgFVq\n+Axr9ssRwN2R7JNSsUbLMkopFYc0c1dKqTgUNTX3goKiOv+FSE9PY8+eoMtkxC0dd2LRcSeWxo47\nI6O1LVB7TGXuTmdtJ9vFLx13YtFxJ5ZwjTumgrtSSqnQaHBXSqk4FFLN3RjzONAP6yzBiSKy1Nfe\nGXit0qbHYK13nYy1sNB6X/snInJ/U3VaKaVU7eoM7saYQUAPEelvjDkOa/nV/lCxtsRg33ZOrIsz\nvA9cAswRkVvC022llFK1CaUskw28CyAieUC6MaZNgO1GA/8Rkf1N1z2llFINEUpZpiOwrNL9Al9b\nYbXtxlL1ogGDjDEfAknALSKyorYXSU9PC+mocUZG6xC6HH903IlFx51YwjHuhsxzrzGn0hjTH1gt\nIv6A/zVQICIf+B57GTih+n6VhTLPMyOjNQUFRfXvcYzTcScWHXdiaey4g/1hCKUsk4+VqftlYl2I\nuLIRWNeWBEBEVovIB77bXwEZvqvmKKWUArZts/Hii0nceisUh+HcrVAy94+BqUCub5W+fBGp/mem\nL77rMAIYY24FtorIbGPM8VhZvJsY8+STjyOSx2+/7aakpITMzM60aXMY06Y9Uue+8+fPo2XLVgwa\ndFbAx2fOnM6ll15BZmbnBvVtwoRx/O1vt3LMMQGvM6GUijLl5bB0qYNPP3Xw6adOVq+28t2kJMjJ\nsXHssU27zledwV1ElhhjlhljlgAe4CZjzGhgn4jM9W3WCeuKMn6vA68YY27wvca1TdrrIObOdTJj\nRjJr1tjJyvIwaVIZOTmuBj/fX/7yV8AK1Bs2rGfChEkh7zts2MhaH5848e8N7pdSKnYUFcHDD6cw\ne3YShYVWVTs11cuQIS6GDHFxxRWppKU1/QKOIdXcRWRytaYfqj1+QrX724DAKWuYzJ3r5PrrW1Tc\nz8tz+O4fbFSAD2T58u94441XKS4uZsKEv7JixTK++OIzPB4P/fsPYMyYcfzrX7kcfvjhHH10d955\n501sNjubN29k8OBsxowZV5F5f/75Zxw4sJ8tWzazffs2br757/TvP4BXX32RTz/9mMzMzjgckJNz\nOX36/L5GX/bv38/999/N/v1FuFwuJk36B8b0ZMaMR1i9Og+3201OziUMGzYyYJtSKjy8XnjnHSd3\n3ZXCL7/Y6dTJwyWXlDNkiIsBA9y08IWrjIxUCsJwJeKoWTissWbMSA7YPnNmcpMHd4D169cxe/Y7\nJCcns2LFMp5++nnsdjuXXXYhl1/+xyrb/vzzT7z++n/weDxceulIxowZV+XxX37ZxaOPPsHXXy/h\nvff+Q+/ex/POO28xe/Z/OHDgAKNGXUxOzuUB+/HWW7Pp3ft4rrpqNKtX/8yTTz7GtGmPsGTJl7z5\n5nu4XC7mz59HYeG+Gm1KqfAQsTN5cgqLFztJTfVy222l3HRTGampzdeHuAnua9YEPjYcrL2xjj22\nB8nJ1h+U1NRUJkwYh8PhYO/evRQWVp0lakxPUmv5VE888XcAtG/fnv3797Nt21aOOaY7KSmppKSk\ncuKJJwbdd/Xqn7nmGqvq1bNnL7Zt20qbNofRpUs3Jk/+G2edNYTzzhtOcnJyjTalVNPavx+mT08h\nNzcJl8vGuee6uO++Erp1a/7rZsTN2jJZWZ56tTdWUlISADt37mDOnNeYPv1JnnrqWTp27FhjW4ej\n9olClR/3er14vWC3H/pobLaAK3pWPFb5gisejzXe6dOf4M9/HsfatWu47ba/Bm1TSjVeYSE88UQy\np57aklmzksnM9PLKK8W88srBiAR2iKPgPmlSWcD2iRMDtzeVvXv3kp6eTlpaGiKr2blzJ+Xl5Y16\nzk6dOrFhw3pcLhd79uxh1apVQbft2bMXK1Z8B8CqVSs5+uju7NiRz1tvvYExPZkwYRL79u0L2KaU\napyCAhv335/MySe34r77UigttXHLLaUsWnSAc8+N7ATBuCnLWHX1g8yceWi2zMSJjZstE4oePbJo\n0SKNG28cwwkn/I4LL7yY6dMf4sQTT2rwcx5xRFuGDj2P6667hm7djubEE08Mmv1fdtkopk2bys03\n34DH4+Fvf7uNdu0yWLXqBz777GOSkpIYPvyCgG1KqYbZtMlGbm4yr72WREmJjXbtrHgzenQZbQIt\nzhIBUXMN1VCuxJRIZ7DNnz+PoUPPw+FwMGbMH3n44Zm0b98h0t1qVon0eVem444eXq8VyFetcvDj\nj3ZWrnSwcqWdggKr6NGli4fx48v44x/LK2a/1FcTnKEasG4bN5l7vNm9ezfjxv2JpKRkRo4cmXCB\nXalI8nphzhwnd9+dwm+/Va1eH3mkh/POK2f4cBcXX+zCd/gt6mhwj1JXXz2aq68eDURnRqNUvNq/\nH269NZW3306idWsvF19czgknuDnhBA/HH+/miCMi3cPQaHBXSimfH3+0c911Ldi40c4pp7h55pnI\nzXZprLiZLaOUUg3l9cKzzyZx/vlpbNxoZ8KEUt5/vzhmAzto5q6USnAbNti4665UPvrISbt2Hp56\n6iBnnx1z6xzWoMFdKZVQvF5YtcrO/PlO5s93kpdnTTMeONDF00+X0KFD7GbrlWlZphbXX/9nVq/O\nq9L2zDNPMXv2qwG3X778O+6441YAJk/+W43H//OfOfzrX7lBX2/durVs2bIZgLvuup3S0pKGdp1L\nLhlJcTgWiVYqyu3aZeObbxwsWuTgs88czJ/v5N13nbzxhrWIV9++LcnObsn06Sls2GDn3HNdPPXU\nQd5882DcBHbQzL1WQ4eey4IFn9Cz53EVbV98sYAnn3ymzn0ffPCxer/ewoUL6NmzF127dmPq1Afq\nvb9Siay0FJ56KpkZM5IpLQ2+ZEerVl5ycqypjGef7aJVq2bsZDPS4F6L7OxzuPHGaxk//mYAVq/O\nIyMjg4yM9ixd+g3PP/8MSUlJtG7dmnvuebDKvsOHZ/PBB5/x3Xff8sQT0zniiLa0bduOzMzOuFwu\n7r//bgoKfuHgwYOMGTOOjh078d5777Bw4QLS09O5887befnlOezfX8Rtt03kwIGD2O12Jk+egs1m\n4/777yYzszPr1q0lK8swefKUgGP45ZddPPDAPZSXl1fs3759B+65Zwq7d/9KWVkZ1157Pb///ak1\n2vr1Oz3s77FSTWHJEgf/+EcKa9c6aN/ew2WXldGiBaSkQHKyl+Rk63anTh4GDHCTkhLpHodfzAT3\nu+9O4YMPwONp2WTPOXKki7vvLg36eHr6EWRmdubnn1fRq9fxLFjwCUOHngdAUVERd911H5mZnbn3\n3jv55puvSEtLq/EcublPMWXKvfTokcUtt9xMZmZniooKOfXUfpx//gi2b9/GlCmTeeGFVznttP4M\nHpxNr17HV+z//PPPcMkll9C370A+//xTXnjhWa699npE8pg6dRrp6UeQkzOMoqIiWreueS3F559/\nhhEjLiQ7+5yK/S+9dBT79u1l1qznKCoq4quvFrN+/boabUpFu927bUydmsIbbyRhs3kZM6aMf/6z\nNGqWAIgkrbnXYejQ8/jss08AWLz4/zF4cDYAhx9+OA89dB8TJoxjxYplFBYGXohrx44d9OiRBcDv\nftcHgNat25CX9xM33jiG+++/O+i+ACJ5nHrqqQD06fN71q4VADp37kLbtu2w2+20a5fBgQP7g+5/\n8smnVNm/W7ejKC4+wL33TmH58qUMGXJOwDaloo3XC3v2wPff2/nXv5IYMCCNN95I4vjj3fz3v8U8\n+KAGdr8YytxLmTUrmYKCA836uoMGncXLL7/A0KHn0qVLV9r4fnIeeOBeHnlkBkcddTSPPfZQ0P0r\nL93rX8fnk08+pLCwkFmznqewsJCxY6+upQeHlvQtL3dhs1nPV30hseBrBNXcPzU1ldzcF1m58kf+\n+995LF68iH/+866AbUpF0rZtNl56KYn16+1s3mx9+S9VB5CW5uWee0oYO7YcZ8xEs+ahb0cd0tJa\n0r17D15++d8VJRmAAwf206FDR4qKili+fBndu/cIuH+7dhls2bKJLl26sWLFMnr3PoG9e/fSqVMm\ndrudhQsXVCwRbLPZcLurzq897rhefPPNN5x22iC+/35ZlYO7oTjuuF4sX/4dQ4eeV7G/yGo2bdrA\nuecOo3fv4xk/fmzANqUipbwccnOTePTRFIqLrWDeooWXbt089O/vpWtXD926eRgxwkVmZvzMcGlK\nGtxDMHToedx3313cdde9FW0XX3wpN954LV26dOXKK6/hhReeZdy48TX2HTduPHfccRsdO3aqWPxr\n8OCzmTz5b/z88yqGD7+A9u3b8+9/P8dJJ53MjBmPVKndjx17A9OnT+O112bjdCZx++1TcLlCX8Z4\n7NgbeOCBe5k3792K/VNSUsnNncV7772D3W7nj3+8mk6dMmu0KRUJ33zj4NZbU8jLc9C2rYdp00rI\nznbTvr2XWq5bo6rRJX9jgI47sSTquG221kycWMbrr1uXr7z66jLuuKOU9PQIdyzMdMlfpVTcKS+H\npUsdLFjg4LXXYPfuZHr1cvPIIyX07RueS2QmCg3uSqlmtXWrjQULnCxY4GDRIif791uJZ6tWMHVq\nCdddpwdHm4K+hUqpZrFmjZ3bbkth8eJDYeeYYzxccUU5Z5/t4oIL0igubtz1h9UhGtyVUmFVUgIz\nZybzxBPJlJfbGDjQxfDhLs46y8XRRx861NayJehySE1Hg7tSKmwWL3Zwyy2prF9vp1MnDw8+WML5\n54f3ovXKElJwN8Y8DvQDvMBEEVnqa+8MvFZp02OAycBbwItAN8AN/FlENjRdt5VS0WzPHpg6NYXX\nX0/GZvNy3XVl3H57adwu0hWN6gzuxphBQA8R6W+MOQ54AegPICLbgcG+7ZzAF8D7wB+BvSJypTHm\nHOAB4PJwDEApFT3WrLHz0ktJzJmTRGGhjd693UyfXkKfPjrzpbmFsrZMNvAugIjkAenGmECrN4wG\n/iMi+337zPW1fwoMaHxXlVLRqKwM3nvPSU5OC844oyXPPZdMaqqXu+4q4eOPizWwR0goZZmOwLJK\n9wt8bYXVthsLnFNpnwIAEfEYY7zGmGQRKWtkf5VSUWL3bhvPPZfEK68kUVBg5YlnnOFi9Ohyzj/f\nRVJShDuY4BpyQLXG2VDGmP7AahGpHvCD7lNdenoaTqejrs3IyKi5rG0i0HEnlmge92+/wWOPwcyZ\nsH8/HH44TJoE118PPXs6acw8jWgedziFY9yhfAr5WJm4Xyawo9o2I7DKL9X3+cEYkwTY6sra9+yp\new5Uop6WreNOLNE67n37IDc3mdzcZIqKbLRv7+H228u48spy/MshFRQ0/Pmjddzh1gTLDwRsDyW4\nfwxMBXKNMX2AfBGp3pO+wBvV9rkU+AgYCXxe3w4rpaLDrl02Xn89iaefTmbfPhvt2nn4+99LGT36\nUFBX0afO4C4iS4wxy4wxSwAPcJMxZjSwT0T8B007Ab9U2m0OMNQY8yVQinWwVSkVI9avtzF/fhLz\n5ztZtswql6ane7njjlLGjCnTKY0xIKTimIhMrtb0Q7XHT6h23w38uXFdU0o1F68XVq2y83//52T+\nfCciVkC3270MGOBi2DAXV1xRToArOaoopWeoKpXA1q+38c47Sbz7rpO1a62Anprq5bzzyhk2zMXQ\noW7ato2OZcFV/WhwVyrBbN9u4913ncydm8SPPx4K6CNHlnPRRS7OPttFy6a7Dr2KEA3uSiWArVtt\nzJvnZN68pIoautPpJTvbRU6ONS9dSy7xRYO7UnHI44GNG218+KEV0JcvP1RDHzjQxciR1peWXOKX\nBnelYtyuXTZ++slOXp6d1asdiNhZs8ZecWFpf0C/4ALrwGhGhgb0RKDBXakYVVIC06alkJubhNd7\n6CTw5GQvPXp4MMbD6ae7GTbMRbt2GtATjQZ3pWLQypV2xo9PRcTBMcd4uOSSMozxcNxxbo46yquX\nqVMa3JWKJS4XPPlkMo88kozLZWPMmDKmTCnV2S2qBg3uSsWIDRts3HRTC5Ytc9Cxo4eZMw9y1lnu\nSHdLRSkN7kpFKY8H8vLsfPONg2+/dfDhh06Ki23k5JTz4IMlpKdHuocqmmlwVyqKbNpk4/nnYcGC\nFnz7rYPCwkMHSjt08PD44yXk5Og1SFXdNLgrFQVcLnjmmSQefjiFkhIAJ0cf7WHYMBennebmtNNc\ndO/uxVbnlRGUsmhwVyrCVq2y89e/pvLDDw7atfMwa5aNU0/dT4cOOn1RNZwGd6UipLQUHn88mSee\nsGa+XHZZOffcU4IxrSko0MCuGkeDu1LN6MAB2LrVztq1dh56KJk1axwceaSHRx89yNln68wX1XQ0\nuCsVJps22XjrrSTWrLGzZYudrVtt/Pqrvco2115bxv/8T6le/EI1OQ3uSjUhlws+/tjJSy8l8cUX\njoplAZKTvRx5pJfevV107eqha1cvZ57p4uSTPRHusYpXGtyVagL5+TZefTWJ115LYscOKzvv29fN\nNdeUceaZbjp08GK31/EkSjUhDe5KNVBBgY0PPnDy/vtOlixx4PHYaN3ay5gxZVxzTTm9emlWriJH\ng7tS9RAooAP8/vdurryynIsuKtd1XlRU0OCuVB1KSuDDD53Mnp3EwoWHAnrfvm4uuKCcESNcdO6s\nUxdVdNHgrlQAXi/88IOd2bOTmDs3ib17rYB+yiluLrpIA7qKfhrclfJxuayAvnChk/fec5KXZ12a\nrn17DxMmlHHFFS6ysrSOrmKDBneVsLxeaxndhQudLFzoYPFiZ8VCXUlJXkaMKGfUqHLOOsutF79Q\nMUd/ZFXCyc+3MWdOEm+8kcTGjYfmJ3br5uHCC8sZPNjNGWe4dEldFdM0uKuEUFYGH33k5PXXk/j8\nc+ugaIsWVnY+eLCbM890cdRRWkNX8UODu4pb+/fDN984WLDAyTvvONm928rSTznFzahR1rTFNm0i\n3EmlwiSk4G6MeRzoB3iBiSKytNJjXYDZQDKwXERuMMYMBt4CfvJttlJE/tKUHVequvJyWLbMwaJF\n1teyZQ7Ky60aetu2Hm64oYxRo8o57jg9KKriX53B3RgzCOghIv2NMccBLwD9K20yHZguInONMbOM\nMV197QtF5JKm77JSh3i98O23Dl57LYl585wcOGAFc7vdy0kneRg40MXAgW7693eTnBzhzirVjELJ\n3LOBdwFEJM8Yk26MaSMihcYYOzAQGOV7/CYAY8wx4eqwUgC//GLjzTetGvq6ddaUxa5dPVx+eTkD\nB7oZMMDF4YdHuJNKRVAowb0jsKzS/QJfWyGQARQBjxtj+gCLROR233a9jDHvA0cAU0Xkk9peJD09\nDafTUWdnMjJah9Dl+KPjhr174aOPYM4cmDfPmpeekgKjRsHYsTB4sB27PT7Sc/28E0s4xt2QA6q2\narc7AzOBTcAHxpjhwPfAVOBN4Bjgc2PMsSJSFuxJ9+wprvOFMzJaU1BQ1IAux7ZEHXe7dq1ZvPgA\nn3zi4NNPnXzzjQO32/rx693bzVVXlXPxxeUVUxZ3745gZ5tQon7eOu6G7x9IKME9HytT98sEdvhu\n/wpsFpH1AMaYz4DeIvIBMMe3zXpjzE6sPwIb6991lUhKSmDJEgeffOJkwQLYuNFahctm89Knj4ch\nQ1ycc46L44/36MWilapFKMH9Y6wsPNdXeskXkSIAEXEZYzYYY3qIyFrgFGC2MeZKoJOIPGqM6Qh0\nALaHaQwqxuXn2/j0UyeffOJk0SIHxcVW1G7TBi64oJwhQ1xkZ7vJyNB56EqFqs7gLiJLjDHLjDFL\nAA9wkzFmNLBPROYCk4AXfQdXVwLzgJbA68aYC7GmSN5YW0lGxS+3G5Yvt/Pll04KCmwUFdkoLMT3\n3cbevTa2bDl0lmiPHm6GDHEzdKiLESPS2Lu3JIK9Vyp2hVRzF5HJ1Zp+qPTYOuCMao8XASMb1zUV\nqwoKbHz+uXXy0OefO9mzJ3D9JC3NS5s2XgYPtkotQ4ZUPUs0Kam5eqxU/NEzVFWjHTwIS5c6+PJL\nBwsXOvn+e3vFtUMzMz2MGFHO2We76drVQ5s2VkBv3RpdjEupMNJfL1VvLhd8/72dRYucfPmlg2+/\ndVBaagVzp9PL6ae7yc626uQ9e+qBT6UiQYO7qtPBg7BihYOvv3bw1VcOli49dNAT4Pjj3Qwc6Gbg\nQBf9+rlp1SqCnVVKARrcVQCVyyxffeVgxQoHZWWHgrkxbvr1c3PmmW5OP91N27Y6i0WpaKPBXVFW\nBsuXW8H8yy8dfPfdoWBut3s54QQP/fpZ67OcdpoGc6VigQb3BFNSAqtX2/nxRwc//mh9z8uzV9TM\nbTYrmA8YYF2wol8/N60T84xwpWKaBvc4deAAbNhgZ906O2vXWt/XrLG+XK5DJZbkZC89e3o49VQ3\nZ5zhpn9/vQKRUvFAg3uMKiqCrVvtbN9uY/t26/u2bXby862TgrZvt9fYp0ULaxncE090c+KJ1ndj\nPLoUrlJxSIN7lHK7Yds2G+vX29m9G376KYUtW2xs3WpnyxZ70BODbDYvHTt6GTjQxbHHeujRw0P3\n7tb3zEwv9poxXykVhzS4R0hZGfz6q42CAutr1y47mzbZWLfOzvr1djZuPFQHt1jpdUqKly5dPJx8\nsvX9yCO9dO5sfc/M9NCpk1fP7FRKaXBvjNJS2LPHxq+/2ti9u+r3PXtsHDxoo7gYDh60cfAgFBfb\nOHAAfv01eOYN0KqVVQfv3t36OumkFA4//ADdunnJyNDsWylVt7gN7l4vFBdDcrJ1mnuoZ0l6PLBz\np43Nm+1s3mxj0yarDPLrr9aiV/v2wb591u2SkvqdetmihZe0NC/t23vo3dsK1O3bW98zMjx06+al\ne3cP7dt7q/Q3IyOFggK97qdSKnQxHdznznXy0EPJbNxoJz3dy9FHe7DbbezaZWPnTlvFXG2bzUty\nsrUQVXKyVbaw262A7//uD6b//C+oAAAQSUlEQVSV96suKcnLYYd5adMGjjzSWifl8MO9tGvnpW3b\nQ98zMqz2li29tGhhLZCVmopm3EqpZhOzwX3uXCfXX9+i4v5vv9n47Tc7drsVfK1A6iUtzVq86rDD\nrDJKeTmUldnweq0sHazvHo+V7ffu7aFbNw9du1qZtP92hw5WgNZ1UpRSsSBmg/uMGYHn73Xs6CU/\n/1CKXFwM69Y5yM09WLHfli02srI8TJpURk6Oq1n6q5RSzSlmCwVr1gTuen5+4NT6nntSuP76FuTl\nWdfhzMtzcP31LZg71/r7Nneuk0GD0ujUqRWDBqXV2a6UUtEsZiNVVpaHvDxHyNtv3x446M+caf0H\nULnE4w/8S5eW8fzzyTXa4dB/AWvW2PW/AKVU1InZ4D5pUlmVgOzXubM3aCAPZM0ae9ASzyuvBJ4w\nfs89KVXOANWgr5SKNjEb3K2AeZCZMw8F0okTrcu01ifoZ2V5EAlc4iktDfzawf541Bb0NcArpZpT\nzNbcwQrwX3xRTH7+fr74opicHBc5OS5ycw/Sq5cbp9NLr15ucnMPcuedgSP1xIllZGUFnkOeklK/\n/tRV+tH6vVKqucRldPEH+ZpqZvr+7QJl+1dfXV6l5u7XkNJP9ambWspRSoVTXAb3YIIF/WAlnpwc\nF337upuk9BOsrq+lHKVUOCRUcK9NbYE/1P8CIHDQnzixjPHjUwO+bl2lHCujh6ysNM3olVIh0+De\nQPUt/cyYUb+pm6tX27WMo5RqMJvXGx3XwywoKKqzIxkZrSkoKGqO7jS56jV3v86dPQEvrJGS4q22\n5G/t2+fmxl8ZJ5Y/78bQcSeWxo47I6N1wH//Y3q2TCyp7yye8vLAz1NbGUdn4yil/PS3vxnVp5Qz\nY0Zyk5Vx4i2jV0rVLaTgbox5HOgHeIGJIrK00mNdgNlYlwpaLiI31LWPqipY0K/PjJykpMAnXc2c\nmUxOjou5c51ap1cqgdRZljHGDAJ6iEh/4FrgiWqbTAemi8ipgNsY0zWEfVQdqpZxaHAZp/Ic+2CL\npiml4k8oNfds4F0AEckD0o0xbQCMMXZgIPC+7/GbRGRLbfuo0PnPwC0vp84zcI0JfJZtbXPstU6v\nVPwK5Te5I7Cs0v0CX1shkAEUAY8bY/oAi0Tk9jr2CSg9PQ2ns+4ac0ZG6xC6HH8qj3vcOOvL4gBa\n0KYNjBpVc78pUxxcdVXg51y92hGwTt+mDVxxRZN1vVH0804sOu6m05A0zVbtdmdgJrAJ+MAYM7yO\nfQLas6e4zhfWqVLBZWdDbq6zxoHZ7GwXWVlpAQ/OJiUFnm55771uCgvLIl6j1887sei4G75/IKEE\n93ysrNsvE9jhu/0rsFlE1gMYYz4DetexjwqTYAdmgy2PHKxOrzNvlIp9odTcPwYuAfCVXvJFpAhA\nRFzABmNMD9+2pwBS2z6q+dW3Tp8UeBn7iiURlFLRr87MXUSWGGOWGWOWAB7gJmPMaGCfiMwFJgEv\n+g6urgTmiYin+j7hG4IKRX2mW9Y18ybS5RqlVN10+YEYEM5xz51bs04f7ASq5l76QD/vxKLjbvD+\nAY9p6ry3BFefjD6YqitYakavVDTQtWVUDcFq9Dt3Bp705D8AqydJKRU9NLirgAJdwjDY5Qj1AKxS\n0UeDuwrZpEllAdtrOwALeu1YpSJBg7sKWUOWPtB1bZSKDP0NU/VSnwOw/pk3gfhXq1RKhYdm7qrR\ngmX0OTmuitJMdf4581quUSo89LdJNYlgGX1WVuBrx3bo4NUlDpQKI83cVVgFOwgbTOVliJ1ONKNX\nqoE0uKuwatycefQArFINpMFdhZ3OmVeq+WlwVxHRkDnzegBWqdBpcFcRUd858/4DsDpfXqnQ6G+G\nipimWrRMZ9coVZNm7iqqVM3oqfMArJZrlApMfwtU1PFn9NY619a1dWfM0PnyStWHZu4qJjRkvrxS\niUyDu4oJ9Z0vrytSqkSnP+kqZgQ6ABusXFN5RUo/LdmoRKKZu4ppwco1da1IqVS80+CuYpquSKlU\nYPoTrWKerkipVE2auau41ZgVKTWjV7FOg7uKW41bkVKXOFCxTYO7imu6IqVKVBrcVcLRFSlVIgjp\nJ9QY8zjQD/ACE0VkaaXHNgFbAbev6UqgB/AW8JOvbaWI/KVpuqxU41gHTQ8yc2Yya9bYycryVEyd\n1AOwKl7UGdyNMYOAHiLS3xhzHPAC0L/aZueLyP5K+/QAForIJU3aW6WaiK5IqeJdKGWZbOBdABHJ\nA9KNMW3C2iulIkCXOFDxJJSfwo7Askr3C3xthZXanjHGHAV8Cdzua+tljHkfOAKYKiKfNL67SoWX\nLnGg4kVDUozqacydwIfAb1gZ/h+Ar4CpwJvAMcDnxphjRSToxOP09DSczpq/QNVlZLRuQJdjn447\ncu68E0aNqtk+ZYqDadMCl3FmzWrBuHENf81oGHck6LibTijBPR8rU/fLBHb474jIy/7bxpj5wAki\n8jYwx9e83hizE+gMbAz2Inv2FNfZEWt976IQuhxfdNyRlZ0NubnOGgdgs7NdXHVVK2rmO/Dzz16e\nfbaEGTMO7TNpUllI2Xy0jLu56bgbvn8goQT3j7Gy8FxjTB8gX0SKAIwxh2Fl5yN9Wfkg4G1jzJVA\nJxF51BjTEegAbG9w75WKMF3iQMWaOg+oisgSYJkxZgnwBHCTMWa0MSZHRPYB84GvjTGLserxbwPv\nA4OMMYuA94AbayvJKBWr9CIiKlqFVHMXkcnVmn6o9NhMYGa1x4uAkY3rmlLRL9ic+fHjUwNu7z8h\nqiHlGqXqQ+dsKdVI9Zlho+Ua1Vx0+QGlwkDLNSrSNLgrFQYNOSHKfzKU04meDKUazeb1eiPdBwAK\nCorq7IhOlUos8TjuQYPSApZrOnf2sH17zVzLf1WpRBCPn3commAqZMCMQTN3pZqRXkBENRcN7ko1\nI72AiGouGtyVamZ6ARHVHDS4KxUFGnIBEdAVKVVwGtyVigJVyzVUlGuMCZzRV16RUks2KhD9KVAq\nSvhPhrJmTxxaSC/QBUT8V44KRC8iokAzd6WiWrADsDk5rorSTHV6zVcFmrkrFfV0RUrVEJq5KxWj\ndIkDVRsN7krFqMYscaDlmvinn6xSMUxXpFTBaOauVJzRJQ4UaHBXKu7oEgcKNLgrFZd0iQOlwV2p\nBNGQJQ60XBO7NLgrlSCClWuCLXHgPwCr5ZrYpMFdqQQSqFzT0PnymtVHNw3uSiW4hs6X16w+umlw\nV0rV6wBsVpan1kXLVHTQ4K6UCihYuWbixDJdtCwGaHBXSgVU24qUwbJ6PQgbPfQdV0oFFWxFykmT\nygKuMx+Mv1wzY0Yya9bYycryMGlSmS57EEaauSul6k3Pgo1+Ib2zxpjHgX6AF5goIksrPbYJ2Aq4\nfU1Xisj22vZRSsW++ixalpQEpaU1n0OvGhU+dWbuxphBQA8R6Q9cCzwRYLPzRWSw72t7iPsopeJM\nY86CdTrRA7BNKJSyTDbwLoCI5AHpxpg2YdhHKRXjGncWLFquaUKhvIMdgWWV7hf42gortT1jjDkK\n+BK4PcR9qkhPT8PprPnvXHUZGa1D6HL80XEnllge97hx1pfFAbSgTRsYNarmtnZ74Pxy1qwWjBsH\nb7wB06bBzz9Dr17wz3/CFVeEq+eRE47PuyF/HqsfMbkT+BD4DStb/0MI+9SwZ09xXZv4rgpfFEIX\n44uOO7HE47izsyE318nMmYdmy0ycWMb48akECg8//+zl2WdLqszIWbnS+gNRWBhfFxdp7Ocd7A9D\nKME9Hyvr9ssEdvjviMjL/tvGmPnACXXto5RKPPU5AFvXWbDxFNzDJZSa+8fAJQDGmD5AvogU+e4f\nZoz5yBjj/xQGAatq20cppfz0LNjwqTO4i8gSYJkxZgnWrJebjDGjjTE5IrIPmA98bYxZjFVbfzvQ\nPuEbglIqVlU9AIueBduEbF6vN9J9AKCgoKjOjsRjLTIUOu7EouO2+FeerK5zZw/bt9fMS3v1cjNx\nYlnMnQXbBDX3gMc09QxVpVRU0rNgG0eDu1IqajXVtWATsUavwV0pFVPqexZsomb0GtyVUjGlvmfB\nJmpGHz8jUUoljGBLEQc6AFtXRu/nz+ghPk6S0sxdKRUXmjKjjweauSul4kZTZPT+k6fmznXG3LTK\nyjRzV0rFtfpm9FlZnoo59rF8EDZ2eqqUUg1Un4zefyJUILF0uUDN3JVSCam2C4AHW9cmlqZVRl+P\nlFKqmQTL6LOy6n+5QIiujF4zd6WUqiYeTpTS4K6UUtXEw4lSWpZRSqkAYv1EKc3clVIqROHI6J1O\nwpLRa+aulFL1ECsZvWbuSinVSNG49IFm7kop1QSacumDpqCZu1JKhUlDlj5oKpq5K6VUGNV36YOm\nopm7Uko1s6oZPVWWPmgqmrkrpVQE+DP6jIzWFBQUN/nza+aulFJxSIO7UkrFIQ3uSikVhzS4K6VU\nHNLgrpRSccjm9Xoj3QellFJNTDN3pZSKQxrclVIqDmlwV0qpOKTBXSml4pAGd6WUikMa3JVSKg5p\ncFdKqTgUE6tCGmMeB/oBXmCiiCyNcJfCyhhzPPAe8LiIPGWM6QK8AjiAHcDVIlIayT6GgzHmYWAg\n1s/lA8BS4nzcxpg04EWgA5AK3Av8QJyP288Y0wJYhTXuz4jzcRtjBgNvAT/5mlYCDxOGcUd95m6M\nGQT0EJH+wLXAExHuUlgZY1oCT2L9oPvdA8wSkYHAOmBMJPoWTsaYs4DjfZ/zecAMEmDcwEjgOxEZ\nBFwGPEZijNvvDuA33+1EGfdCERns+/oLYRp31Ad3IBt4F0BE8oB0Y0ybyHYprEqBYUB+pbbBwPu+\n2/OAIc3cp+bw/4BLfbf3Ai1JgHGLyBwRedh3twuwjQQYN4AxpifQC/jA1zSYBBh3AIMJw7hjoSzT\nEVhW6X6Br60wMt0JLxFxAS5jTOXmlpX+TfsF6NTsHQszEXEDB3x3rwXmA+fG+7j9jDFLgCOBEcCn\nCTLu6cAE4E+++3H/c+7TyxjzPnAEMJUwjTsWMvfqbJHuQITF9fiNMRdiBfcJ1R6K63GLyOnABcCr\nVB1rXI7bGHMN8JWIbAyySVyOG1iLFdAvxPqj9i+qJtlNNu5YCO75WJm6XybWQYdEst934AmgM1VL\nNnHDGHMu8D/A+SKyjwQYtzHmFN8Bc0Tke6xf9KJ4HzcwHLjQGPM1MBaYQgJ83iKy3VeK84rIemAn\nVqm5yccdC8H9Y+ASAGNMHyBfRIoi26Vm9ynwB9/tPwAfRrAvYWGMOQx4BBghIv4DbHE/buBM4O8A\nxpgOQCsSYNwicrmI9BWRfsDzWLNl4n7cxpgrjTG3+G53xJol9W/CMO6YWPLXGPMg1i+BB7hJRH6I\ncJfCxhhzClYt8iigHNgOXIk1XS4V2Az8WUTKI9TFsDDGjAPuBtZUav4T1i9+PI+7Bda/5l2AFlj/\nsn8HvEwcj7syY8zdwCbgI+J83MaY1sDrwOFAMtbnvYIwjDsmgrtSSqn6iYWyjFJKqXrS4K6UUnFI\ng7tSSsUhDe5KKRWHNLgrpVQc0uCulFJxSIO7UkrFof8PcV/kafCwDhYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f9ae2e0aba8>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}