text-generation-with-rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# import necessary modules\n",
    "from __future__ import print_function\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.models import Sequential \n",
    "from keras.utils.vis_utils import plot_model\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read input text and cleanup content\n",
    "fin = open(\"../Learn-enough-python-and-machine-learning-to-dangerous/data/code.txt\", \"rb\")\n",
    "lines = []\n",
    "for line in fin:\n",
    "    line = line.strip().lower()\n",
    "    line = line.decode(\"ascii\",\"ignore\")\n",
    "    if len(line) == 0:\n",
    "        continue\n",
    "    line += '\\n'\n",
    "    lines.append(line)\n",
    "fin.close()\n",
    "text = \" \".join(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70\n",
      "{'(', 'o', '-', 'i', '7', '|', ']', ':', ',', '\\\\', 'l', '%', '@', 'v', '3', '!', '/', '^', 'e', 'w', 'q', 'r', 'a', '*', '6', '[', '2', 'u', 'z', '?', '4', '~', '<', 'b', '\"', 'd', 't', '.', 'p', ' ', 'y', '0', 'c', 'g', 'h', '1', 'x', '\\n', '$', '&', '5', '+', '_', '>', '{', 's', '9', '#', '8', 'k', 'f', 'j', ';', ')', '`', 'm', \"'\", 'n', '=', '}'}\n"
     ]
    }
   ],
   "source": [
    "# build vocabulary\n",
    "chars = set([c for c in text])\n",
    "nb_chars = len(chars)\n",
    "print(nb_chars)\n",
    "print(chars)\n",
    "char2index = dict((c, i) for i, c in enumerate(chars))\n",
    "index2char = dict((i, c) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the input and label texts  \n",
    "SEQLEN = 10\n",
    "STEP = 1\n",
    "\n",
    "input_chars = []\n",
    "label_chars = []\n",
    "for i in range(0, len(text) - SEQLEN, STEP):\n",
    "    input_chars.append(text[i:i + SEQLEN])\n",
    "    label_chars.append(text[i + SEQLEN])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectorize these input and label texts\n",
    "X = np.zeros((len(input_chars), SEQLEN, nb_chars), dtype=np.bool)\n",
    "y = np.zeros((len(input_chars), nb_chars), dtype=np.bool)\n",
    "for i, input_char in enumerate(input_chars):\n",
    "    for j, ch in enumerate(input_char):\n",
    "        X[i, j, char2index[ch]] = 1\n",
    "    y[i, char2index[label_chars[i]]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_1 (LSTM)                (None, 256)               334848    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 70)                17990     \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 70)                0         \n",
      "=================================================================\n",
      "Total params: 352,838\n",
      "Trainable params: 352,838\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# build our model\n",
    "HIDDEN_SIZE = 256\n",
    "BATCH_SIZE = 128\n",
    "NUM_ITERATIONS = 25\n",
    "NUM_EPOCHS_PER_ITERATION = 2\n",
    "NUM_PREDS_PER_EPOCH = 500\n",
    "\n",
    "model = Sequential()\n",
    "model.add(LSTM(HIDDEN_SIZE, return_sequences=False, input_shape=(SEQLEN, nb_chars), unroll=True))\n",
    "model.add(Dense(nb_chars))\n",
    "model.add(Activation(\"softmax\"))\n",
    "model.summary()\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"rmsprop\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "Iteration #: 0\n",
      "Epoch 1/2\n",
      "637891/637891 [==============================] - 325s 510us/step - loss: 2.1805\n",
      "Epoch 2/2\n",
      "637891/637891 [==============================] - 330s 517us/step - loss: 1.5827\n",
      "classmmmmm]()222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222==================================================\n",
      "Iteration #: 1\n",
      "Epoch 1/2\n",
      "637891/637891 [==============================] - 327s 513us/step - loss: 1.3854\n",
      "Epoch 2/2\n",
      "637891/637891 [==============================] - 349s 546us/step - loss: 1.2757\n",
      "classm#////###kk   //.///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////==================================================\n",
      "Iteration #: 2\n",
      "Epoch 1/2\n",
      "637891/637891 [==============================] - 341s 534us/step - loss: 1.2009\n",
      "Epoch 2/2\n",
      "637891/637891 [==============================] - 331s 518us/step - loss: 1.1452s - loss: 1.\n",
      "classx*##h#####thkkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdkkdk==================================================\n",
      "Iteration #: 3\n",
      "Epoch 1/2\n",
      "354688/637891 [===============>..............] - ETA: 2:29 - loss: 1.096"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-197544e3662c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"=\"\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Iteration #: %d\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0miteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNUM_EPOCHS_PER_ITERATION\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[1;31m#test_idx = np.random.randint(len(input_chars))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1039\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    202\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m                 \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    205\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mcallback_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    113\u001b[0m         \u001b[0mt_before_callbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 115\u001b[1;33m             \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    116\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mt_before_callbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    117\u001b[0m         \u001b[0mdelta_t_median\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    328\u001b[0m         \u001b[1;31m# will be handled by on_epoch_end.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseen\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 330\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_values\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    331\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    332\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_epoch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\keras\\utils\\generic_utils.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, current, values)\u001b[0m\n\u001b[0;32m    354\u001b[0m             \u001b[0mprev_total_width\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_total_width\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    355\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dynamic_display\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 356\u001b[1;33m                 \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'\\b'\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mprev_total_width\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    357\u001b[0m                 \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'\\r'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    358\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, string)\u001b[0m\n\u001b[0;32m    398\u001b[0m             \u001b[0mis_child\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_master_process\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    399\u001b[0m             \u001b[1;31m# only touch the buffer in the IO thread to avoid races\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 400\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    401\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_child\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    402\u001b[0m                 \u001b[1;31m# newlines imply flush in subprocesses\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m    201\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m             \u001b[1;31m# wake event thread (message content is ignored)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 203\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    204\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m             \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\zmq\\sugar\\socket.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, data, flags, copy, track, routing_id, group)\u001b[0m\n\u001b[0;32m    390\u001b[0m                                  copy_threshold=self.copy_threshold)\n\u001b[0;32m    391\u001b[0m             \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 392\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSocket\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrack\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    393\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    394\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0msend_multipart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmsg_parts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._send_copy\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\envs\\tf3\\lib\\site-packages\\zmq\\backend\\cython\\checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for iteration in range(NUM_ITERATIONS):\n",
    "    print(\"\\n\")\n",
    "    print(\"=\"*50)\n",
    "    print(\"Iteration #: %d\" % (iteration))\n",
    "    model.fit(X, y, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS_PER_ITERATION)\n",
    "    \n",
    "    test_idx = np.random.randint(len(input_chars))\n",
    "    test_chars = input_chars[test_idx]\n",
    "    print(\"Generating from seed: %s\" % (test_chars))\n",
    "    print(test_chars, end=\"\")\n",
    "    for i in range (NUM_PREDS_PER_EPOCH):\n",
    "        Xtest = np.zeros((1, SEQLEN, nb_chars))\n",
    "        for i, ch in enumerate(test_chars):\n",
    "            Xtest[0, i, char2index[ch]] = 1\n",
    "        pred = model.predict(Xtest, verbose=0)[0]\n",
    "        ypred = index2char[np.argmax(pred)]\n",
    "        print(ypred, end=\"\")\n",
    "        test_chars = test_chars[1:] + ypred\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}