new_speech2string_without_masking_with_Lasagne_CTC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports\n",
    "======"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/theano/tensor/signal/downsample.py:6: UserWarning: downsample module has been moved to the theano.tensor.signal.pool module.\n",
      "  \"downsample module has been moved to the theano.tensor.signal.pool module.\")\n"
     ]
    }
   ],
   "source": [
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne \n",
    "from lasagne.layers import InputLayer, DenseLayer, RecurrentLayer, NonlinearityLayer, ReshapeLayer, get_output, get_all_params, get_all_param_values, ElemwiseSumLayer\n",
    "import ctc_cost\n",
    "from time import time\n",
    "from TIMIT_utils import index2char_TIMIT\n",
    "from special_activations import clipped_relu\n",
    "import sys\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data\n",
    "========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = open('TIMIT_data_prepared_for_CTC.pkl','rb')\n",
    "data = pkl.load(f)\n",
    "inp = data['x']\n",
    "inp1 = data['inputs']\n",
    "msk = data['mask']\n",
    "tgt = data['y_indices']\n",
    "char = data['chars']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the network\n",
    "================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_size = len(inp1[0][0])\n",
    "hidden_size = 50\n",
    "num_output_classes = len(char)\n",
    "learning_rate = 0.001\n",
    "output_size = num_output_classes+1\n",
    "batch_size = None\n",
    "input_seq_length = None\n",
    "gradient_clipping = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Introduce the targets\n",
    "====================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = T.imatrix('targets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the Bi-RNN architecture\n",
    "=============================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "l_in = InputLayer(shape=(batch_size, input_seq_length, input_size))\n",
    "n_batch, n_time_steps, n_features = l_in.input_var.shape #Unnecessary in this version. Just collecting the info so that we can reshape the output back to the original shape\n",
    "# h_1 = DenseLayer(l_in, num_units=hidden_size, nonlinearity=clipped_relu)\n",
    "l_rec_forward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu)\n",
    "l_rec_backward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu, backwards=True)\n",
    "l_rec_accumulation = ElemwiseSumLayer([l_rec_forward,l_rec_backward])\n",
    "l_rec_reshaped = ReshapeLayer(l_rec_accumulation, (-1,hidden_size))\n",
    "l_h2 = DenseLayer(l_rec_reshaped, num_units=hidden_size, nonlinearity=clipped_relu)\n",
    "l_out = DenseLayer(l_h2, num_units=output_size, nonlinearity=lasagne.nonlinearities.linear)\n",
    "l_out_reshaped = ReshapeLayer(l_out, (n_batch, n_time_steps, output_size))#Reshaping back\n",
    "l_out_softmax = NonlinearityLayer(l_out, nonlinearity=lasagne.nonlinearities.softmax)\n",
    "l_out_softmax_reshaped = ReshapeLayer(l_out_softmax, (n_batch, n_time_steps, output_size))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the outputs\n",
    "==============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "output_logits = get_output(l_out_reshaped)\n",
    "output_softmax = get_output(l_out_softmax_reshaped)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect all the parameters\n",
    "=========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_params = get_all_params(l_out,trainable=True)\n",
    "# print all_params==[l_rec.W_in_to_hid, l_rec.b, l_rec.W_hid_to_hid, l_out.W, l_out.b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trainable parameters = 10\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print 'Number of trainable parameters =', len(all_params)\n",
    "print all_params==[l_rec_forward.W_in_to_hid, l_rec_forward.b, l_rec_forward.W_hid_to_hid, l_rec_backward.W_in_to_hid, l_rec_backward.b, l_rec_backward.W_hid_to_hid, l_h2.W, l_h2.b, l_out.W, l_out.b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute cost\n",
    "============"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pseudo_cost = ctc_cost.pseudo_cost(y, output_logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute gradients\n",
    "================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pseudo_cost_grad = T.grad(pseudo_cost.sum() / n_batch, all_params)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute cost for evaluation\n",
    "==========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_cost = ctc_cost.cost(y, output_softmax)\n",
    "cost = T.mean(true_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate parameter updates\n",
    "==========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shared_learning_rate = theano.shared(lasagne.utils.floatX(0.01))\n",
    "updates = lasagne.updates.rmsprop(pseudo_cost_grad, all_params, learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the training op\n",
    "======================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "theano.config.exception_verbosity='high'\n",
    "train = theano.function([l_in.input_var,y], [output_logits, output_softmax, cost, pseudo_cost], updates=updates)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sanity check the input data\n",
    "==========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 278, 39) (1, 52)\n"
     ]
    }
   ],
   "source": [
    "inp0 = inp1[0]\n",
    "inp00= np.asarray([inp0],dtype=theano.config.floatX)\n",
    "tgt0 = np.asarray(tgt[0],dtype=np.int16)\n",
    "tgt00 = np.asarray([tgt0])\n",
    "print inp00.shape, tgt00.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run Training\n",
    "============"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Cost:  695.258914005 , time taken = 0.464868068695\n",
      "Predicted: ______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _________________________________________________________________________________________________________________________________________________________\n",
      "Target: i saw your horse outside\n",
      "Epoch:  1 Cost:  342.928594065 , time taken = 0.416613101959\n",
      "Epoch:  2 Cost:  179.489504172 , time taken = 0.405362129211\n",
      "Epoch:  3 Cost:  170.353690783 , time taken = 0.440174102783\n",
      "Epoch:  4 Cost:  163.192402963 , time taken = 0.405067920685\n",
      "Epoch:  5 Cost:  156.962595127 , time taken = 0.410233020782\n",
      "Epoch:  6 Cost:  150.732229863 , time taken = 0.410019159317\n",
      "Epoch:  7 Cost:  144.213916376 , time taken = 0.402567863464\n",
      "Epoch:  8 Cost:  137.57019083 , time taken = 0.404509067535\n",
      "Epoch:  9 Cost:  131.081858182 , time taken = 0.413440942764\n",
      "Epoch:  10 Cost:  124.487711411 , time taken = 0.414126157761\n",
      "Predicted: ____________________________________________________________o_______________________ ___________________________________ _________________________________________________________________a____ _____________at_______________________________________________________________________\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _________________________________________ ___a________________________________________________a___ _____________________a________________________________\n",
      "Target: i saw your horse outside\n",
      "Epoch:  11 Cost:  117.754391567 , time taken = 0.447767972946\n",
      "Epoch:  12 Cost:  110.836461606 , time taken = 0.428574085236\n",
      "Epoch:  13 Cost:  105.898919285 , time taken = 0.416472911835\n",
      "Epoch:  14 Cost:  107.680783819 , time taken = 0.422326803207\n",
      "Epoch:  15 Cost:  106.533532606 , time taken = 0.419183969498\n",
      "Epoch:  16 Cost:  100.035885964 , time taken = 0.418770074844\n",
      "Epoch:  17 Cost:  87.1408354663 , time taken = 0.420788049698\n",
      "Epoch:  18 Cost:  79.5227705391 , time taken = 0.425096035004\n",
      "Epoch:  19 Cost:  73.4768237044 , time taken = 0.41675901413\n",
      "Epoch:  20 Cost:  68.2099057461 , time taken = 0.411364793777\n",
      "Predicted: _______________________________________________ _____y______ou__ d__________________ _________________________uu__i____t i______________________ ______________re_______a______y_____ ___aa___h ______w______at________________ __a_ __l _______________________________e_a___________\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re__a___ar____a_______________________r_ ___ _______________________________________ ________a___ ____r ______r________ _______ _______________ ________\n",
      "Target: i saw your horse outside\n",
      "Epoch:  21 Cost:  63.5831362397 , time taken = 0.421214103699\n",
      "Epoch:  22 Cost:  59.3020642967 , time taken = 0.406807899475\n",
      "Epoch:  23 Cost:  55.4734078076 , time taken = 0.422957181931\n",
      "Epoch:  24 Cost:  52.488289869 , time taken = 0.420311927795\n",
      "Epoch:  25 Cost:  48.9821892326 , time taken = 0.412256002426\n",
      "Epoch:  26 Cost:  45.9154323602 , time taken = 0.408935070038\n",
      "Epoch:  27 Cost:  40.6925957131 , time taken = 0.417082071304\n",
      "Epoch:  28 Cost:  35.7549082236 , time taken = 0.404206037521\n",
      "Epoch:  29 Cost:  31.5326324974 , time taken = 0.412347793579\n",
      "Epoch:  30 Cost:  27.8280499631 , time taken = 0.414273023605\n",
      "Predicted: sh___ h____a___________________________________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re__a___ar____a____________________g__r_r___ _____________________e_________________ ________a___ ____r ______r_g____g_ _______ _______________ _s______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  31 Cost:  24.8733769688 , time taken = 0.417771100998\n",
      "Epoch:  32 Cost:  22.337057492 , time taken = 0.412835121155\n",
      "Epoch:  33 Cost:  20.3623817962 , time taken = 0.427484989166\n",
      "Epoch:  34 Cost:  18.4281522935 , time taken = 0.414781093597\n",
      "Epoch:  35 Cost:  17.1687421698 , time taken = 0.417280912399\n",
      "Epoch:  36 Cost:  15.6881284507 , time taken = 0.424530982971\n",
      "Epoch:  37 Cost:  14.9465407564 , time taken = 0.4105489254\n",
      "Epoch:  38 Cost:  13.4430649052 , time taken = 0.410006046295\n",
      "Epoch:  39 Cost:  12.6493225725 , time taken = 0.416280984879\n",
      "Epoch:  40 Cost:  11.4918678028 , time taken = 0.414304971695\n",
      "Predicted: sh__e h____a___________________________________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_____e rh___a____________________g__r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_ _____ _ _______________ ls______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  41 Cost:  10.4982937984 , time taken = 0.420031070709\n",
      "Epoch:  42 Cost:  9.33870641798 , time taken = 0.401381015778\n",
      "Epoch:  43 Cost:  8.5642309527 , time taken = 0.39977812767\n",
      "Epoch:  44 Cost:  7.6467728752 , time taken = 0.40585398674\n",
      "Epoch:  45 Cost:  7.02417568064 , time taken = 0.409202814102\n",
      "Epoch:  46 Cost:  6.21159696292 , time taken = 0.412712812424\n",
      "Epoch:  47 Cost:  5.79530629742 , time taken = 0.412620782852\n",
      "Epoch:  48 Cost:  5.10275867153 , time taken = 0.4261739254\n",
      "Epoch:  49 Cost:  4.70748777423 , time taken = 0.417473077774\n",
      "Epoch:  50 Cost:  4.2161527069 , time taken = 0.418891906738\n",
      "Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_____e rh___a_______________________r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_y_____ _ ________________gs______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  51 Cost:  3.92090622112 , time taken = 0.4078810215\n",
      "Epoch:  52 Cost:  3.53588133573 , time taken = 0.411890983582\n",
      "Epoch:  53 Cost:  3.27719304612 , time taken = 0.410335063934\n",
      "Epoch:  54 Cost:  2.99158167047 , time taken = 0.411561965942\n",
      "Epoch:  55 Cost:  2.80235490122 , time taken = 0.416101932526\n",
      "Epoch:  56 Cost:  2.57297723797 , time taken = 0.407879114151\n",
      "Epoch:  57 Cost:  2.41938355212 , time taken = 0.401879787445\n",
      "Epoch:  58 Cost:  2.22885560338 , time taken = 0.399511098862\n",
      "Epoch:  59 Cost:  2.1265490806 , time taken = 0.399022102356\n",
      "Epoch:  60 Cost:  1.92352310151 , time taken = 0.407444953918\n",
      "Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_h___e rh___a_______________________r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_y_____ _ ________________gs______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  61 Cost:  1.80484279124 , time taken = 0.411237001419\n",
      "Epoch:  62 Cost:  1.65171381186 , time taken = 0.398931980133\n",
      "Epoch:  63 Cost:  1.56257511647 , time taken = 0.423418998718\n",
      "Epoch:  64 Cost:  1.41884125149 , time taken = 0.407498121262\n",
      "Epoch:  65 Cost:  1.3246180081 , time taken = 0.403403043747\n",
      "Epoch:  66 Cost:  1.21549038112 , time taken = 0.416229963303\n",
      "Epoch:  67 Cost:  1.14304921779 , time taken = 0.412912845612\n",
      "Epoch:  68 Cost:  1.05509660723 , time taken = 0.417524814606\n",
      "Epoch:  69 Cost:  0.995153813729 , time taken = 0.406607151031\n",
      "Epoch:  70 Cost:  0.921401154805 , time taken = 0.402260065079\n",
      "Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _______________ ________a___ _____ _____wg_g____g_y_____ _ ________________gs______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  71 Cost:  0.869646956452 , time taken = 0.445788145065\n",
      "Epoch:  72 Cost:  0.803373195665 , time taken = 0.420474052429\n",
      "Epoch:  73 Cost:  0.758740814343 , time taken = 0.416527032852\n",
      "Epoch:  74 Cost:  0.702201068917 , time taken = 0.415088891983\n",
      "Epoch:  75 Cost:  0.664818493953 , time taken = 0.413058996201\n",
      "Epoch:  76 Cost:  0.611278627215 , time taken = 0.415685176849\n",
      "Epoch:  77 Cost:  0.576907684605 , time taken = 0.417546987534\n",
      "Epoch:  78 Cost:  0.534437784294 , time taken = 0.419013023376\n",
      "Epoch:  79 Cost:  0.508590999047 , time taken = 0.415677070618\n",
      "Epoch:  80 Cost:  0.466608813544 , time taken = 0.421269893646\n",
      "Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _l_____________ ________a___ _____ _____wg_g____g_y_______ ________________gs______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  81 Cost:  0.440707552881 , time taken = 0.417589902878\n",
      "Epoch:  82 Cost:  0.407251542954 , time taken = 0.422451972961\n",
      "Epoch:  83 Cost:  0.386693420567 , time taken = 0.424895048141\n",
      "Epoch:  84 Cost:  0.358527518907 , time taken = 0.417057991028\n",
      "Epoch:  85 Cost:  0.339120320275 , time taken = 0.421405076981\n",
      "Epoch:  86 Cost:  0.313970282001 , time taken = 0.413981199265\n",
      "Epoch:  87 Cost:  0.297925901041 , time taken = 0.424841880798\n",
      "Epoch:  88 Cost:  0.276489075815 , time taken = 0.41254401207\n",
      "Epoch:  89 Cost:  0.264080459513 , time taken = 0.415322065353\n",
      "Epoch:  90 Cost:  0.24550561974 , time taken = 0.410289049149\n",
      "Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____\n",
      "Target: she had your dark suit in greasy wash water all year\n",
      "Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _l_____________ ________a___ _____ _____wg_g____g_y____a _ ________________gs______\n",
      "Target: i saw your horse outside\n",
      "Epoch:  91 Cost:  0.234048768906 , time taken = 0.406760931015\n",
      "Epoch:  92 Cost:  0.216640805133 , time taken = 0.412416934967\n",
      "Epoch:  93 Cost:  0.205755959314 , time taken = 0.412768125534\n",
      "Epoch:  94 Cost:  0.189948261922 , time taken = 0.423375844955\n",
      "Epoch:  95 Cost:  0.181333456368 , time taken = 0.409070968628\n",
      "Epoch:  96 Cost:  0.167968282014 , time taken = 0.401962995529\n",
      "Epoch:  97 Cost:  0.160850249575 , time taken = 0.412121772766\n",
      "Epoch:  98 Cost:  0.14895047728 , time taken = 0.411957979202\n",
      "Epoch:  99 Cost:  0.142852759076 , time taken = 0.41521692276\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 100\n",
    "# num_training_samples = len(inp1)\n",
    "num_training_samples = 1\n",
    "for epoch in range(num_epochs):\n",
    "    t = time()\n",
    "    cost = 0\n",
    "    failures = []\n",
    "    for i in range(num_training_samples):\n",
    "        curr_inp = inp1[i]\n",
    "#         curr_msk = msk[i].astype(np.bool)\n",
    "#         curr_inp = curr_inp[curr_msk]\n",
    "        curr_inp = np.asarray([curr_inp],dtype=theano.config.floatX)\n",
    "        curr_tgt = np.asarray(tgt[i],dtype=np.int16)\n",
    "        curr_tgt = np.asarray([curr_tgt])\n",
    "        try:\n",
    "            _,_,c,_=train(curr_inp,curr_tgt)\n",
    "            cost += c\n",
    "        except IndexError:\n",
    "            failures.append(i)\n",
    "            print 'Current input seq: ', curr_inp\n",
    "            print 'Current output seq: ', curr_tgt\n",
    "            sys.exit(IndexError)\n",
    "    print 'Epoch: ', epoch, 'Cost: ', float(cost/(num_training_samples-len(failures))), ', time taken =', time()-t\n",
    "#     print 'Exceptions: ', len(failures), 'Total examples: ', num_training_samples\n",
    "    if epoch%10==0:        \n",
    "        #Save the model\n",
    "        np.savez('CTC_model_under_test.npz', *get_all_param_values(l_out_softmax_reshaped, trainable=True))\n",
    "        for i in range(2):\n",
    "            curr_inp = inp1[i]\n",
    "            curr_inp = np.asarray([curr_inp],dtype=theano.config.floatX)\n",
    "            curr_tgt = np.asarray(tgt[i],dtype=np.int16)\n",
    "            curr_out = output_softmax.eval({l_in.input_var:curr_inp})\n",
    "            print 'Predicted:', index2char_TIMIT(np.argmax(curr_out, axis=2)[0])\n",
    "            print 'Target:', index2char_TIMIT(curr_tgt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}