Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "tK8UcPKzXxOh",
    "outputId": "c491d3d1-9f90-44ae-9afb-5fa4b22703cd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "ZHf32YwKdb2u",
    "outputId": "9e838671-7528-4440-ba68-a4bb1fbd569b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (1) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "import ast\n",
    "image_path = '/ImageCap/imagesB/flickr30k-images/'\n",
    "feat_path = '/ImageCap/image_feat/'\n",
    "text_path = '/ImageCap/captions_pros.csv'\n",
    "df = pd.read_csv(text_path)\n",
    "\n",
    "image_name_list = list(set(df['image_name'])) #obtaining unique instance name of each image\n",
    "image_path_list = list(map(lambda arg: image_path + arg, image_name_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "v4k6Wd0-2ULg",
    "outputId": "7c40f5d8-f8bc-45e3-fa2e-f037673dc23e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18290, 300)"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M = np.load('/ImageCap/embedB.npy')\n",
    "M.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "2S0aYfudo8O-",
    "outputId": "c6bdde9e-5b5c-47e0-cc8e-902cd099357b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158915, 80)"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cap_seq = np.load('/ImageCap/caption_vec.npy')\n",
    "cap_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Cqb1OmTnR8hi"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "word_ind_map=dict()\n",
    "with open('/ImageCap/word_ind_map.pkl', 'rb') as f:\n",
    "  word_ind_map = pickle.load(f)\n",
    "\n",
    "ind_word_map=dict()\n",
    "with open('/ImageCap/ind_word_map.pkl', 'rb') as f:\n",
    "  ind_word_map = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "bvKDGcaRB_9M",
    "outputId": "38292ce4-1266-4c10-aa6a-3f8f58e9ff9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000 100000 10000 10000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "#Since there are multiple captions for same images so in order to have disjoint train and test sets\n",
    "#(sharing no images) we need to first split and then shuffle \n",
    "#(unlike what's done by train_test_split() function of sklearn)\n",
    "\n",
    "#num below should be a multiple of 5 because for each image there are 5 captions which means \n",
    "#same image is repeated 5 times\n",
    "\n",
    "num = 100000 #num of training examples \n",
    "test_num = 10000 #num of test examples\n",
    "\n",
    "X = np.array(df['image_name'])\n",
    "\n",
    "X_train = X[0:num]\n",
    "y_train = cap_seq[0:num]\n",
    "X_train, y_train = shuffle(X_train, y_train, random_state=1)\n",
    "\n",
    "X_test = X[num:num+test_num]\n",
    "y_test = cap_seq[num:num+test_num]\n",
    "X_test, y_test = shuffle(X_test, y_test, random_state=1)\n",
    "\n",
    "print(len(X_train), len(y_train), len(X_test), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QyGXR-601BNu"
   },
   "outputs": [],
   "source": [
    "def map_to_img(img_name, caption):\n",
    "  img = np.load(feat_path + img_name.decode('utf-8') + '.npy')\n",
    "  return img, caption\n",
    "\n",
    "data = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n",
    "data = data.map(lambda arg1, arg2: tf.numpy_function(func = map_to_img, inp = [arg1, arg2], Tout = [tf.float32, tf.int32]), num_parallel_calls=tf.data.experimental.AUTOTUNE) \n",
    "\n",
    "data = data.shuffle(1000).batch(64)\n",
    "data = data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OwEziQH16Uyz"
   },
   "outputs": [],
   "source": [
    "class ENCODER(tf.keras.Model):\n",
    "  def __init__(self, embed_dim):\n",
    "    super(ENCODER,self).__init__()\n",
    "    self.den = tf.keras.layers.Dense(embed_dim)\n",
    "\n",
    "  def call(self, x): #input x will be of dimention (batch x 64 x 2048), this can be visulized as 64 \n",
    "                     #attentionable regions in image each with 2048 features\n",
    "    x = self.den(x)  #This dense is applied so as to make feature space of image regions (2048 features) \n",
    "                     #and that of tokens (300 features) consistent \n",
    "    x = tf.nn.relu(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NsWutIQih1Fg"
   },
   "outputs": [],
   "source": [
    "class ATTEND(tf.keras.Model): #this class deals with training of a small NN to learn score which is \n",
    "                              #further used to calculate attention weights and context\n",
    "  \n",
    "  def __init__(self, units):\n",
    "    super(ATTEND, self).__init__()\n",
    "    #Here W1 and W2 are used to learn suitable parameters so as to obtain suitable \n",
    "    #score=tanh(W1(feat)+W2(hidden)) (which minimizes cost) while V is used to apply \n",
    "    #softmax on score (along dim of attentionable regions (64)) so as to obtain \n",
    "    #attention weights for all attentionable regions \n",
    "    \n",
    "    self.W1 = tf.keras.layers.Dense(units)\n",
    "    self.W2 = tf.keras.layers.Dense(units)\n",
    "    self.V = tf.keras.layers.Dense(1)\n",
    "\n",
    "  def call(self, feat, hidden): #where score depends on features computed by encoder (feat) \n",
    "                                #and hidden state upto current time of decoder (hidden)\n",
    "                                #where feat is of dim (batch x 64 x 300) and hidden is of \n",
    "                                #dim (batch x hidden_size) \n",
    "   \n",
    "    hidden_ = tf.expand_dims(hidden, axis=1) #This basically inserts a new axis at axis \n",
    "                                             #index 1 (it is being done so that addition of \n",
    "                                             #W1 and W2 while calculating score can be done properly \n",
    "                                             #by broadcasting axis 1 to 64 units as shown below)\n",
    "    \n",
    "    score = tf.nn.tanh( self.W1(feat) + self.W2(hidden_) ) #score will be of dim (batch x 64 x hidden_size)\n",
    "\n",
    "    att_wt = tf.nn.softmax(self.V(score), axis=1) #Note that softmax is applied at axis=1 \n",
    "                                                  #(which contains all attentionable regions)\n",
    "                                                  #att_wt is of dim (batch x 64 x 1)\n",
    "\n",
    "    #Context is basically weighted sum of features (weighted by attention weights), so here \n",
    "    #we first perform mult of all 300 features by att_weights (via broadcasting)\n",
    "    #Then summation is done along the dimention of attentionable regions (i.e axis of 64 (here axis=1))\n",
    "    \n",
    "    context = att_wt*feat \n",
    "    context = tf.reduce_sum(context, axis=1) #its dim will be (batch x hidden_size)\n",
    "\n",
    "    return context, att_wt\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "35NE03jax--2"
   },
   "outputs": [],
   "source": [
    "class DECODER(tf.keras.Model):\n",
    "\n",
    "  def __init__(self, units, embed_M, sentence_length):\n",
    "    super(DECODER, self).__init__()\n",
    "    self.units = units\n",
    "    self.embed = tf.keras.layers.Embedding(input_dim=embed_M.shape[0], output_dim=embed_M.shape[1], weights=[embed_M], input_length=sentence_length, trainable=False )\n",
    "    self.lstm = tf.keras.layers.LSTM(units=units, return_sequences=True, return_state=True) \n",
    "    #at each time step, lstm is supposed to return both output as well as hidden state\n",
    "    \n",
    "    #output of lstm is fed to a FC network consisting of 2 dense layers\n",
    "    self.den1 = tf.keras.layers.Dense(units)\n",
    "    self.den2 = tf.keras.layers.Dense(embed_M.shape[0])   \n",
    "    #This last dense layer contains vocab no of elements (it just contains elements and not their probability\n",
    "    # (tanh activation is being used, not softmax) .Later for loss we will simply find difference \n",
    "    #in probability distribution of actual and predicted elements using entropy)\n",
    "\n",
    "    self.attend = ATTEND(units)\n",
    "\n",
    "\n",
    "\n",
    "  def call(self, tok, feat, hidden): #decoder takes prev time-step token, hidden state of prev \n",
    "                                     #time-step decoder RNN and context vector for current time-step \n",
    "                                     #(which is obtained by attedding over image feat) as input and \n",
    "                                     #returns tok and hidden state at current time-step\n",
    " \n",
    "    context, att_wt = self.attend(feat, hidden)\n",
    "    x = self.embed(tok) #obtain token in embedded form (batch x 1 x embed_dim)\n",
    "\n",
    "    #Now we need to feed both embedded tok and context as input to the lstm layer, \n",
    "    #but lstm layer has single input arg, so we concatenate both these things and input this concat to lstm\n",
    "    context_ = tf.expand_dims(context,1) #insert a new axis at index 1 so that upcoming concat can happen properly\n",
    "    x = tf.concat([context_, x], axis=2) #concat will happen along axis=2 leading to \n",
    "                                         #dim (batch x 1 x (embed_dim+hidden_size))\n",
    "    output, state, _ = self.lstm(x) \n",
    "    \n",
    "    x = self.den1(output) #(batch x sentence_length x hidden_size)\n",
    "    x = tf.reshape(x, (-1, x.shape[2])) # this reduces first 2 dims to single dim \n",
    "                                        #by mult them to give ((batch*sentence_length) x hidden_size)\n",
    "    x = self.den2(x) \n",
    "    return x, state, att_wt\n",
    "\n",
    "  def reset_state(self, batch_size): #hidden state dim is (batch x units) and is fixed for each time-step \n",
    "    return tf.zeros((batch_size, self.units))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Xkz98G9wB9pi"
   },
   "outputs": [],
   "source": [
    "embed_dim = 300\n",
    "units = 512\n",
    "version = '5' #different model versions were trained using different amount of training data. Of these,\n",
    "              #version 5 (trained on 20,000 images and corrosponding captions) performs the best\n",
    "              #This Notebook contains training code for the same\n",
    "\n",
    "enc = ENCODER(embed_dim)\n",
    "dec = DECODER(units, M, len(y_train[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "C10_xa0IB9ro"
   },
   "outputs": [],
   "source": [
    "loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "\n",
    "def loss_fun(real, pred):\n",
    "  mask = tf.math.logical_not(tf.math.equal(real,0)) #If real == padding_token then put 0 in mask vector else 1 \n",
    "  L = loss_obj(real, pred) \n",
    "  mask = tf.cast(mask, dtype=L.dtype) #by default mask will be of bool type, \n",
    "                                      #so before applying it we need to make its type same as that of loss\n",
    "  L *= mask #This elementwise mult with mask removes loss for padding tokens\n",
    "  return tf.reduce_mean(L) #mean of loss vector is returned\n",
    "\n",
    "opt = tf.keras.optimizers.Adam()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "5uKEhCZfB9wX",
    "outputId": "6ec83180-5308-412d-e2df-2cbddaa1dc54"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint_path = \"/ImageCap/checkpoints/train\"+version\n",
    "ckpt = tf.train.Checkpoint(encoder=enc, decoder=dec, optimizer = opt) \n",
    "\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
    "start_epoch = 0\n",
    "\n",
    "# restoring the latest checkpoint in checkpoint_path\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "  start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])\n",
    "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "start_epoch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_plot = []\n",
    "try:\n",
    "    #if resuming from an epoch then load list from a saved txt file\n",
    "    with open('/ImageCap/checkpoints/train'+version+'/loss.txt', 'r') as f:\n",
    "          loss_plot = list(map(lambda arg: float(arg), f.readlines()))\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KnjmlzqLM38y"
   },
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train(img, target_cap): #This function is called for each batch\n",
    "  batch_loss = 0\n",
    "  hidden = dec.reset_state(batch_size=target_cap.shape[0])\n",
    "\n",
    "  input_tok = tf.expand_dims([word_ind_map['<start>']]*target_cap.shape[0],axis=1) \n",
    "  #this effectively generates a vector of dim 64x1 having 64 index elements corrosponding to\n",
    "  #<start> token. This is done because the first ever input token fed to decoder is <start> token\n",
    "\n",
    "  with tf.GradientTape() as tape:\n",
    "    feat = enc(img)\n",
    "\n",
    "    for t in range(1,target_cap.shape[1]): #This is infact forward prop through various time steps of decoder RNN \n",
    "                                           #we start with t=1 and not t=0 because input_tok, \n",
    "                                           #hidden etc for t=0 were already obtained above\n",
    "      \n",
    "      pred, hidden, p = dec(input_tok, feat, hidden) #hidden fed on right is of time t-1 and hidden obtained \n",
    "                                                     #on left is of time t\n",
    "      input_tok = tf.expand_dims(target_cap[:,t], axis=1) #Here instead of feeding predicted tok to \n",
    "                                                          #next timestep, we are feeding real tok \n",
    "                                                          #to next timestep. \n",
    "                                                          #This is called teacher forcing \n",
    "      batch_loss += loss_fun(target_cap[:,t], pred) \n",
    "\n",
    "    tot_loss = batch_loss/int(target_cap.shape[1]) #dividing by no of time-steps\n",
    "\n",
    "    trainable_variables = enc.trainable_variables + dec.trainable_variables\n",
    "    gradients = tape.gradient(batch_loss, trainable_variables)\n",
    "    opt.apply_gradients(zip(gradients, trainable_variables))\n",
    "\n",
    "    return batch_loss, tot_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "cPmwkLKrXHfD",
    "outputId": "0730709c-54b8-4fd8-bb88-867ead043d86"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 Batch 0 Loss 0.2638\n",
      "Epoch 18 Batch 100 Loss 0.3131\n",
      "Epoch 18 Batch 200 Loss 0.2853\n",
      "Epoch 18 Batch 300 Loss 0.2691\n",
      "Epoch 18 Batch 400 Loss 0.2876\n",
      "Epoch 18 Batch 500 Loss 0.2938\n",
      "Epoch 18 Batch 600 Loss 0.2800\n",
      "Epoch 18 Batch 700 Loss 0.2929\n",
      "Epoch 18 Batch 800 Loss 0.2849\n",
      "Epoch 18 Batch 900 Loss 0.2615\n",
      "Epoch 18 Batch 1000 Loss 0.2657\n",
      "Epoch 18 Batch 1100 Loss 0.2763\n",
      "Epoch 18 Batch 1200 Loss 0.2514\n",
      "Epoch 18 Batch 1300 Loss 0.2593\n",
      "Epoch 18 Batch 1400 Loss 0.2792\n",
      "Epoch 18 Batch 1500 Loss 0.2992\n",
      "Epoch 18 Loss 0.279881\n",
      "Time taken for 1 epoch 11479.909207582474 sec\n",
      "\n",
      "Epoch 19 Batch 0 Loss 0.2641\n",
      "Epoch 19 Batch 100 Loss 0.2564\n",
      "Epoch 19 Batch 200 Loss 0.2683\n",
      "Epoch 19 Batch 300 Loss 0.2799\n",
      "Epoch 19 Batch 400 Loss 0.3054\n",
      "Epoch 19 Batch 500 Loss 0.2925\n",
      "Epoch 19 Batch 600 Loss 0.2589\n",
      "Epoch 19 Batch 700 Loss 0.2429\n",
      "Epoch 19 Batch 800 Loss 0.2586\n",
      "Epoch 19 Batch 900 Loss 0.2781\n",
      "Epoch 19 Batch 1000 Loss 0.3036\n",
      "Epoch 19 Batch 1100 Loss 0.2616\n",
      "Epoch 19 Batch 1200 Loss 0.2689\n",
      "Epoch 19 Batch 1300 Loss 0.2496\n",
      "Epoch 19 Batch 1400 Loss 0.3013\n",
      "Epoch 19 Batch 1500 Loss 0.3093\n",
      "Epoch 19 Loss 0.278953\n",
      "Time taken for 1 epoch 9043.87217783928 sec\n",
      "\n",
      "Epoch 20 Batch 0 Loss 0.2860\n",
      "Epoch 20 Batch 100 Loss 0.2476\n",
      "Epoch 20 Batch 200 Loss 0.2831\n",
      "Epoch 20 Batch 300 Loss 0.2872\n",
      "Epoch 20 Batch 400 Loss 0.2637\n",
      "Epoch 20 Batch 500 Loss 0.2673\n",
      "Epoch 20 Batch 600 Loss 0.2575\n",
      "Epoch 20 Batch 700 Loss 0.2717\n",
      "Epoch 20 Batch 800 Loss 0.2198\n",
      "Epoch 20 Batch 900 Loss 0.2529\n",
      "Epoch 20 Batch 1000 Loss 0.2876\n",
      "Epoch 20 Batch 1100 Loss 0.2349\n",
      "Epoch 20 Batch 1200 Loss 0.2725\n",
      "Epoch 20 Batch 1300 Loss 0.2473\n",
      "Epoch 20 Batch 1400 Loss 0.2644\n",
      "Epoch 20 Batch 1500 Loss 0.2340\n",
      "Epoch 20 Loss 0.273424\n",
      "Time taken for 1 epoch 10373.156922101974 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 20 \n",
    "num_steps = len(X_train) // 64 #no of steps in a single Epoch\n",
    "for epoch in range(start_epoch, EPOCHS):\n",
    "    start = time.time()\n",
    "    total_loss = 0\n",
    "    #batch=1\n",
    "    \n",
    "    for (batch, (img_tensor, target)) in enumerate(data):\n",
    "        batch_loss, t_loss = train(img_tensor, target)\n",
    "        total_loss += t_loss\n",
    "\n",
    "        if batch % 100 == 0:\n",
    "          print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, batch_loss.numpy() / int(target.shape[1])))\n",
    "\n",
    "    loss_plot.append(total_loss / num_steps)\n",
    "    with open('/ImageCap/checkpoints/train'+version+'/loss.txt', 'a') as f:\n",
    "      f.write(str(total_loss / num_steps))\n",
    "      f.write('\\n')\n",
    "      \n",
    "    ckpt_manager.save()\n",
    "\n",
    "    print ('Epoch {} Loss {:.6f}'.format(epoch + 1, total_loss/num_steps))\n",
    "    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "colab_type": "code",
    "id": "muXZlmm9tew8",
    "outputId": "49dbca3c-c700-4d13-a83f-e974417406be"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhV5bn38e+diUAgIZBEIAQCJICgMkVUBotYFa2FVttq69tqtcfaSq2t7ak97Tltbd9zOtfag/W1rba1VbTaAbV1BhVHEogok4Q5CCYMIYwZ7/ePvaAp7kAg2Xsl2b/Pde0re69hrzsrO/nlWc9azzJ3R0RE5GhJYRcgIiKdkwJCRESiUkCIiEhUCggREYlKASEiIlEpIEREJCoFhEgnY2bfNrM/hF2HiAJCEpqZbTSz94ew3d+aWb2Z7TOzXWb2tJmNPon3CaV+SQwKCJHw/NDdewODgSrgt+GWI/KvFBAiUZhZDzO73czeCR63m1mPYF6OmT1mZjXBf/8vmllSMO9rZrbVzPaa2RozO/9423L3A8D9wGmt1DLbzFYE21tkZqcG0+8DhgCPBi2Rf++o718EFBAirfkGcDYwHhgHTAa+Gcy7BagEcoFTgP8A3MxGAXOBM929D3ARsPF4GzKz3sBVwLIo80YCDwA3B9v7O5FASHP3TwKbgQ+6e293/+FJf7ciUSggRKK7CrjN3avcvRr4DvDJYF4DMBAY6u4N7v6iRwY1awJ6AGPMLNXdN7r7umNs4ytmVgNUAL2Ba6IscwXwuLs/7e4NwI+BnsCUDvgeRY5JASES3SBgU4vXm4JpAD8i8kf9KTNbb2a3Arh7BZH/9L8NVJnZfDMbROt+7O593X2Au89uJUz+pQ53bwa2APkn+X2JtJkCQiS6d4ChLV4PCabh7nvd/RZ3Hw7MBr58uK/B3e9392nBug78oCPrMDMDCoCtwSQNxywxo4AQgVQzS2/xSCFy3P+bZpZrZjnAfwF/ADCzS82sKPhjvYfIoaVmMxtlZjODzuxDwEGguZ21PQR8wMzON7NUIv0fdcDLwfx3geHt3IZIVAoIkUjH78EWj28D3wNKgeXAm8DSYBpAMfAMsA94BbjT3RcS6X/4PrAD2A7kAV9vT2Huvgb4P8Avgvf9IJFO6fpgkf8hEmQ1ZvaV9mxL5GimGwaJiEg0akGIiEhUCggREYlKASEiIlEpIEREJKqUsAvoKDk5OV5YWBh2GSIiXUpZWdkOd8+NNq/bBERhYSGlpaVhlyEi0qWY2abW5ukQk4iIRKWAEBGRqBQQIiISlQJCRESiUkCIiEhUCggREYlKASEiIlElfEBsrTnID59YzbY9B8MuRUSkU0n4gNhf18idi9bxwtvVYZciItKpJHxAFOf15pTMHry4dkfYpYiIdCoJHxBmxtSiHF5et5PmZt08SUTksIQPCIDpxTns2l/Pym21YZciItJpKCCAqUU5ADrMJCLSggICyOuTzugBfVhcoY5qEZHDFBCBaUU5LNm4m0MNTWGXIiLSKSggAtOKc6hvbOb1DbvCLkVEpFOIaUCY2SwzW2NmFWZ2a5T515hZtZmVB4/PtJh3tZmtDR5Xx7JOgLOG9SctOYnFFeqHEBGBGN5RzsySgXnABUAlsMTMFrj7yqMWfdDd5x61bj/gW0AJ4EBZsO7uWNXbMy2ZSUOz1VEtIhKIZQtiMlDh7uvdvR6YD8xp47oXAU+7+64gFJ4GZsWoziOmFeewalst1XvrYr0pEZFOL5YBkQ9safG6Mph2tMvNbLmZPWxmBSeyrpldb2alZlZaXd3+M5CmF0dOd315nVoRIiJhd1I/ChS6+xlEWgm/O5GV3f1udy9x95Lc3Nx2FzN2UBZ9e6XqMJOICLENiK1AQYvXg4NpR7j7Tnc/fDzn18Cktq4bC8lJxtQROSxeuwN3DbshIoktlgGxBCg2s2FmlgZcCSxouYCZDWzxcjawKnj+JHChmWWbWTZwYTAt5qYV57C99hDrqvfFY3MiIp1WzM5icvdGM5tL5A97MnCPu68ws9uAUndfANxkZrOBRmAXcE2w7i4z+y6RkAG4zd3jcoHCtBbDbhTl9YnHJkVEOiXrLodSSkpKvLS0tEPea8aPFjIitze/uebMDnk/EZHOyszK3L0k2rywO6k7pWnFOby6ficNTc1hlyIiEhoFRBTTinLZX9/Ess01YZciIhIaBUQU54zoT5LB4rUa3VVEEpcCIoqsnqmMK+jLixqXSUQSmAKiFdOLcnhjSw17DjaEXYqISCgUEK2YVpxLs8Mr63aGXYqISCgUEK2YMKQvGWnJusuciCQsBUQrUpOTOHt4fxZrXCYRSVAKiGOYVpzDxp0H2LLrQNiliIjEnQLiGA4P/627zIlIIlJAHMOI3N4MyEzXYSYRSUgKiGMwM6YV5/DSuh00NXePMatERNpKAXEc04tzqDnQwIp39oRdiohIXCkgjmNqi+G/RUQSiQLiOHJ69+DUgZnqhxCRhKOAaIPpxTmUbdrNwfqmsEsREYkbBUQbTCvKob6pmdc2aNgNEUkcCog2mDysH2kpSTrMJCIJRQHRBumpyZxZmK0L5kQkoSgg2mhaUS6rt++lau+hsEsREYkLBUQbHR524yW1IkQkQSgg2mjMwEz6ZaTpeggRSRgKiDZKSjKmjIgM/+2uYTdEpPtTQJyA6cU5VO2tY23VvrBLERGJOQXECZhWnAto2A0RSQwKiBOQ37cnw3MyWLxWtyEVke5PAXGCphXn8NqGXdQ3NoddiohITCkgTtC0ohwO1DexdPPusEsREYkpBcQJOntEf5KTTMNuiEi3p4A4QZnpqYwv6MuLumBORLo5BcRJmFaUw5uVNew50BB2KSIiMaOAOAnTinNodnh5nVoRItJ9xTQgzGyWma0xswozu/UYy11uZm5mJcHrQjM7aGblweOuWNZ5osYX9KV3jxQdZhKRbi0lVm9sZsnAPOACoBJYYmYL3H3lUcv1Ab4IvHbUW6xz9/Gxqq89UpOTOHt4P3VUi0i3FssWxGSgwt3Xu3s9MB+YE2W57wI/ALrUONrTinLYvOsAm3ceCLsUEZGYiGVA5ANbWryuDKYdYWYTgQJ3fzzK+sPMbJmZPW9m06NtwMyuN7NSMyutro7v1c1Hht2o0FXVItI9hdZJbWZJwE+BW6LM3gYMcfcJwJeB+80s8+iF3P1udy9x95Lc3NzYFnyUEbkZDMxK1/0hRKTbimVAbAUKWrweHEw7rA9wGrDIzDYCZwMLzKzE3evcfSeAu5cB64CRMaz1hJkZ04pyeKliJ03NGv5bRLqfWAbEEqDYzIaZWRpwJbDg8Ex33+PuOe5e6O6FwKvAbHcvNbPcoJMbMxsOFAPrY1jrSZlWnMOegw28tXVP2KWIiHS4mAWEuzcCc4EngVXAQ+6+wsxuM7PZx1n9XGC5mZUDDwM3uPuuWNV6sqYWRW5DuliHmUSkG7Lucne0kpISLy0tjft2L/n5i2T2TGH+9efEfdsiIu1lZmXuXhJtnq6kbqfpxTmUbdrNgfrGsEsREelQCoh2ml6cS0OT89jybWGXIiLSoRQQ7XT28H5MGprNbY+u1EVzItKtKCDaKSU5iduvGI8Z3DR/GQ1NutOciHQPCogOUNCvF/9z2emUb6nh9mfeDrscEZEOoYDoIJeeMYgrSgq4c9E6XtZpryLSDSggOtC3Zo9hWE4GX3qonF3768MuR0SkXRQQHahXWgp3XDmB3fsb+PeH36C7XGMiIolJAdHBTsvP4msXj+aZVVXc9+qmsMsRETlpCogYuHZqITNG5fK9x1exaltt2OWIiJwUBUQMmBk//ug4MtNTuemBZRysbwq7JBGRE6aAiJGc3j346cfGsbZqH999fOXxVxAR6WQUEDF07shcPnvucO5/bTNPvKWhOESka1FAxNgtF47ijMFZfO2RN3mn5mDY5YiItJkCIsbSUpL4+ZUTaGhq5uYHy3X3ORHpMhQQcTAsJ4PvzjmN1zfsYt7CirDLERFpEwVEnFw2MZ854wdx+zNvU7qx090cT0TkPRQQcWJmfO9Dp5Gf3ZMvzi9nz8GGsEsSETkmBUQc9UlP5Y4rJ/Bu7SH+489vaigOEenUFBBxNmFINl+6YCSPv7mNh0q3hF2OiEirFBAhuOF9I5gyoj/fXrCSiqp9YZcjIhKVAiIEyUnGz64YT3pqEl94YBmHGjQUh4h0PgqIkJySmc6PPjKOVdtq+cETq8MuR0TkPRQQIXr/mFO4+pyh3PvSRv66bGvY5YiI/IuUsAtIdF+/5FTWvLuXLz9UjhnMGZ8fdkkiIoBaEKFLT03mnmvO5MzCfnzpwXIefeOdsEsSEQEUEJ1Cr7QU7v30mZQU9uPmB8t5bLlCQkTCp4DoJHqlpXDvNWcyaUg2X5xfzuPLNTy4iIRLAdGJZPSItCQmDunLTfOXKSREJFQKiE4mEhKTmVAQCYm/v6mQEJFwKCA6od49UvjttZGQ+MIDy/iHQkJEQqCA6KQOh8T4ICSeeGt72CWJSIKJaUCY2SwzW2NmFWZ26zGWu9zM3MxKWkz7erDeGjO7KJZ1dla9e6Tw20+fyRmDs5h7/1KFhIjEVZsCwswyzCwpeD7SzGabWepx1kkG5gEXA2OAj5vZmCjL9QG+CLzWYtoY4EpgLDALuDN4v4TTJz2V3107mdODkHhyhUJCROKjrS2IF4B0M8sHngI+Cfz2OOtMBircfb271wPzgTlRlvsu8APgUItpc4D57l7n7huAiuD9EtLhkDgtP4sb/7iUpxQSIhIHbQ0Ic/cDwGXAne7+USL/3R9LPtDyhgeVwbR/vqnZRKDA3R8/0XWD9a83s1IzK62urm7bd9JFZaan8vvrgpC4fylPr3w37JJEpJtrc0CY2TnAVcDhP+btOuQTHLL6KXDLyb6Hu9/t7iXuXpKbm9uecrqEwyExZlAWn/9jGc8oJEQkhtoaEDcDXwf+4u4rzGw4sPA462wFClq8HhxMO6wPcBqwyMw2AmcDC4KO6uOtm7Ay01P5/bWTGTMwk8/9sYxnVykkRCQ22hQQ7v68u8929x8E//nvcPebjrPaEqDYzIaZWRqRTucFLd5zj7vnuHuhuxcCrwKz3b00WO5KM+thZsOAYuD1E//2uqesnqn8/rqzOHVgJp/7w1KeW62QEJGO19azmO43s0wzywDeAlaa2VePtY67NwJzgSeBVcBDQevjNjObfZx1VwAPASuBJ4Ab3V23XWshq2cq9117FqMG9OGG+5ZqWA4R6XDm7sdfyKzc3ceb2VXAROBWoMzdz4h1gW1VUlLipaWlYZcRd3sONHDt75ZQtmk3t1wwkrkzizCzsMsSkS7CzMrcvSTavLb2QaQG1z18CFjg7g3A8ZNFYi6rVyp//MxZfHhCPj95+m1ufrBc97gWkQ7R1oD4f8BGIAN4wcyGArWxKkpOTHpqMj/92Di+etEo/lb+Dh//1atU760LuywR6eLa2kl9h7vnu/slHrEJOC/GtckJMDNuPK+IX141kVXbavnQvJdYvV0ZLiInr62d1Flm9tPDF6WZ2U+ItCakk7n49IH86bNTaGxu5vI7X9ZpsCJy0tp6iOkeYC/wseBRC9wbq6KkfU4fnMXfbpzG8NzefOb3pfz6xfW05WQEEZGW2hoQI9z9W8G4Suvd/TvA8FgWJu0zICudhz57DrPGDuB7j6/i639+k/rG5rDLEpEupK0BcdDMph1+YWZTgYOxKUk6Ss+0ZOZ9YiJzzyti/pItfOqe16g5UB92WSLSRbQ1IG4A5pnZxmBYjP8FPhuzqqTDJCUZX7loFD+7YhxLN9XwoXkvsa56X9hliUgX0NazmN5w93HAGcAZ7j4BmBnTyqRDfXjCYB64/iz2Hmrkw/NeYvHaHWGXJCKd3AndUc7da9398LmTX45BPRJDk4b24683TmVgVk+uvvd1/vDqprBLEpFOrD23HNV4Dl1QQb9ePPy5c3jfyFy++de3+M6jK2hsUue1iLxXewJC5012UX3SU/nVp0q4btow7n1pI5/5fSl7DjaEXZaIdDLHDAgz22tmtVEee4FBcapRYiA5yfjPS8fw3x8+ncVrd/CBO15k2ebdYZclIp3IMQPC3fu4e2aURx93T4lXkRI7nzhrCA/dcA4AH73rFe56fh3NzWocikj7DjFJNzFxSDaP3zSdC8acwvf/sZprfruEHfs02J9IolNACBC5AdGdV03k/374NF5bv5OLf/6iToUVSXAKCDnCzLjqrKH8be5Usnqm8sl7XuNHT67WWU4iCUoBIe8xekAmC+ZO5YqSAuYtXMcVd7/K1hqNrCKSaBQQElWvtBS+f/kZ3PHxCazZvpeLb3+BJ97aHnZZIhJHCgg5ptnjBvH4TdMozMnghj+U8Z9/fUu3NBVJEAoIOa6h/TN4+IYp/Nv0Ydz36iY+NO8lKqo04J9Id6eAkDZJS0niGx8Yw73XnEnV3jo++IvF/Kl0i25EJNKNKSDkhJw3Oo9/fHE64wv68tWHl/OlB8vZV9cYdlkiEgMKCDlhp2Sm84fPnMUtF4xkwRvvcOkdL/La+p1hlyUiHUwBISclOcn4wvnFzL/+HBqanCvufpUvPLCMd3Q6rEi3oYCQdpk8rB/PfPl93Pz+Yp5asZ3zf/I8v3h2rc50EukGFBDSbj3Tkrn5/SN59pb3cd7oXH7y9Ntc8LPneXLFdnVii3RhCgjpMIOze3HnVZO4/zNn0TM1mc/eV8an7nmdiqq9YZcmIidBASEdbkpRDn+/aTrfmT2WN7bUMOv2F/nuYyupPaSbEol0JQoIiYmU5CSunlLIwq/M4GNnFnDPSxs470eLeHDJZt1vQqSLUEBITPXv3YP//vDpPDp3GsNyMvjaI2/yoTtfomyT7l4n0tkpICQuTsvP4k83nMPPrxzPu7WHuPyXL/Plh8qpqj0Udmki0goFhMSNmTFnfD7P3TKDz88YwWNvbOO8Hy/irufXUd+oe06IdDYxDQgzm2Vma8yswsxujTL/BjN708zKzWyxmY0Jphea2cFgermZ3RXLOiW+Mnqk8O+zRvPUl87lnBE5fP8fq7no9hdYuLoq7NJEpAWL1XnqZpYMvA1cAFQCS4CPu/vKFstkuntt8Hw28Hl3n2VmhcBj7n5aW7dXUlLipaWlHfgdSLwsWlPFbY+tZH31fmaOzuM/Lx3DsJyMsMsSSQhmVubuJdHmxbIFMRmocPf17l4PzAfmtFzgcDgEMgCd3pKAZozK44kvnss3LjmV1zfs4sKfPc/3/7FagwCKhCyWAZEPbGnxujKY9i/M7EYzWwf8ELipxaxhZrbMzJ43s+nRNmBm15tZqZmVVldXd2TtEmdpKUn827nDee4r72PO+Hzuen4dM3+8iL8sq9TV2CIhCb2T2t3nufsI4GvAN4PJ24Ah7j4B+DJwv5llRln3bncvcfeS3Nzc+BUtMZPXJ50ff3Qcf/n8FAZmpfOlB9/gI3e9wpuVe8IuTSThxDIgtgIFLV4PDqa1Zj7wIQB3r3P3ncHzMmAdMDJGdUonNGFINn/5/FR++JEz2LRzP7PnLebWR5azY19d2KWJJIxYBsQSoNjMhplZGnAlsKDlAmZW3OLlB4C1wfTcoJMbMxsOFAPrY1irdEJJScbHSgp47iszuG7qMB4uq+S8Hy/insUbaGjSabEisRazgHD3RmAu8CSwCnjI3VeY2W3BGUsAc81shZmVEzmUdHUw/VxgeTD9YeAGd98Vq1qlc8tMT+Wbl47hiZsjd7K77bGVXPLzF3mpYkfYpYl0azE7zTXedJprYnB3nl75Lt99fCVbdh1k1tgBfOMDp1LQr1fYpYl0Scc6zTUl3sWItIeZceHYAZw7Mpdfv7ieeQvXsXBNFZ88eyjXnzucvMz0sEsU6TbUgpAubdueg/zoyTX8rfwdkpOMK0oK+Oz7hjM4Wy0KkbY4VgtCASHdwuadB/jl8+t4uGwL7nDZxHw+N6NIV2SLHIcCQhLGOzUHufuF9Tzw+mYampr54LhB3HheESNP6RN2aSKdkgJCEk713jp+vXg9972yiQP1TcwaO4C5M4s4LT8r7NJEOhUFhCSs3fvrufelDdz78kb2HmpkxqhcvjCziElD+4VdmkinoICQhFd7qIH7XtnEbxZvYNf+es4Z3p8vzCzinBH9MbOwyxMJjQJCJHCgvpH7X9vM3S+sp2pvHROH9OULM4uZMSpXQSEJSQEhcpRDDU38qaySuxatY2vNQUad0odrpxUyZ3w+6anJYZcnEjcKCJFW1Dc287fyrfxm8QZWb99L/4w0rjp7KJ88eyi5fXqEXZ5IzCkgRI7D3Xll3U5+s3gDz66uIi05idnjB3HdtGGcOvA9I82LdBsaakPkOMyMKUU5TCnKYX31Pu59aSMPl1XycFklU0b057ppwzhvVB5JSeqnkMShFoRIK2oO1PPA61v43csb2V57iOE5GXx6aiGXTxpMrzT9byXdgw4xibRDQ1Mzf39zG/cs3sAblXvI6pnKxycP4eopQxmY1TPs8kTaRQEh0gHcnbJNu/nN4g08uWI7SWZccvpArps2jHEFfcMuT+SkqA9CpAOYGSWF/Sgp7MeWXQf47csbeXDJFha88Q6n5Wdy+cTBzB43iP69dfaTdA9qQYi0w95DDTxSVsnDSyt5a2stKUnGjFF5fGRSPueNzqNHiq6pkM5Nh5hE4mDN9r08srSSvyzbSvXeOvr2SuWDZwzi8kmDGTc4S1dqS6ekgBCJo8amZhZX7OCRpVt5asV26hqbGZGbwWUTB3PZxHx1bEunooAQCUntoQb+vnwbjyytZMnG3ZjB1BE5XDYxn1mnDdDpshI6BYRIJ7Bp537+vHQrf15WyZZdB8lIS+bi0wdy2cR8zh7WXxfhSSgUECKdSHOzs2TjLv68dCuPv7mNfXWN5PftyeUT87l80mCG9tdtUiV+FBAindTB+iaeWrmdh8sqWVyxA3c4szCbj0wazCWnD6RPemrYJUo3p4AQ6QK27TnIX5Zt5ZGyStZV7yc9NYlZYwdw+aTBTBmRQ7IOQUkMKCBEuhB3p3xLDY8srWRB+TvUHmpkYFY6H54QOQQ1Ird32CVKN6KAEOmiDjU08cyqd3mkrJLn366m2WHCkL58ZNJgLj1jEFk9dQhK2kcBIdINVNUe4q/lW3m4rJK3391HWkoSF445hcsnDWZ6UQ4pyUlhlyhdkAJCpBtxd97aWssjSyv5a/lWag40kJmewoxReZx/ah7vG5lL315pYZcpXYQCQqSbqm9sZtGaKp5e+S4L11SxY189SQYlQ/sx89Q83n9qHiNye2uYD2mVAkIkATQ3O8u37uHZVe/y7KoqVm6rBWBIv17MHB1pXUwe1k8DCMq/UECIJKBtew7y3OoqnltVxeKKHdQ1NpORlsy5I3OZOTqP80bnkaOhyROeAkIkwR2sb+LldTt4NgiM7bWHMINxg/ty/ug8Zp6ax5iBmToUlYBCCwgzmwX8HEgGfu3u3z9q/g3AjUATsA+43t1XBvO+DlwXzLvJ3Z881rYUECJt4+6seKeW51ZX8ezqKt7YUgPAgMx0zhudy3mj8phalENGDw0kmAhCCQgzSwbeBi4AKoElwMcPB0CwTKa71wbPZwOfd/dZZjYGeACYDAwCngFGuntTa9tTQIicnKq9h1i0ppqFq6t4ce0O9tU1kpacxFnD+zFzdB4zR+dpfKhuLKxbjk4GKtx9fVDEfGAOcCQgDodDIAM4nFZzgPnuXgdsMLOK4P1eiWG9Igkpr086Hysp4GMlBdQ3NlO6cVek72JNFd95dCXfeXQlw3MzmDkqEhYlhf1IS9E1F4kglgGRD2xp8boSOOvohczsRuDLQBows8W6rx61bn6Uda8HrgcYMmRIhxQtksjSUpKYUpTDlKIcvnnpGDbt3B8Ji9VV/P6VTfx68QZ690hhenEO543OY8aoXPL6pIddtsRI6AcZ3X0eMM/MPgF8E7j6BNa9G7gbIoeYYlOhSOIa2j+DT08dxqenDmN/XSMvVexg4ZoqFq6u5h9vbQfgjMFZTC3KYfKwfkwamk2mRqDtNmIZEFuBghavBwfTWjMf+OVJrisiMZbRI4ULxw7gwrEDcHdWbqtl4eoqFq6p5lcvrOeXi9aRZDBmUCaTC/szeVg/zizMpr9Ope2yYtlJnUKkk/p8In/clwCfcPcVLZYpdve1wfMPAt9y9xIzGwvczz87qZ8FitVJLdI5HahvpHxzDa9t2MXrG3axdPNu6hqbASjK683kYf04a1g/zizsx6C+uid3ZxJKJ7W7N5rZXOBJIqe53uPuK8zsNqDU3RcAc83s/UADsJvg8FKw3ENEOrQbgRuPFQ4iEq5eaSlH+i4gMgTIm1treH3Dbl7fsJNHy9/h/tc2AzA4u+eRwJg8rD+F/Xvp+otOShfKiUjMNTU7q7bV8nrQwnh94y527a8HILdPD0qGZjNpaDYlhf0YOyiTVI1MGze6klpEOhV3Z131/iAwdrJk42621hwEID01iTMG96VkaDYlhdlMHJKt0WljSAEhIp3e9j2HKNu0m9JNu1i6aTcr3qmlsTny96kor/eRVsakodkMy8nQYakOooAQkS7nYH0T5VtqWLp5N6Ubd1G2aTe1hxoB6J+RxsSh2UdC47T8LNJTNUrtyQjrSmoRkZPWMy2Zc0b055wR/YHIcOYV1fsirYyNuynbtIunV74LQGqyMWZgJhOGZDO+oC8ThvRlSD91freXWhAi0mXt2FdH2abdLNtcQ/mW3Syv3MOB+sgJj/0y0iJhUdCX8UP6Mq6gry7ii0ItCBHplnJ69+CisQO4aOwAABqbmnn73X2Ub6lh2ebdlG+p4bnVVQCYwYjc3kcCY0JBNiNP6a17eR+DWhAi0q3tOdjA8soayjfXsGxLDeVbao6cYtsrLZnT87OYMCSbCUMirY28zMQaW0otCBFJWFk9U5lenMv04lwgcort5l0HglZGpKXxm8XraWiK/LOc37dn0MLoy4Qh2YwdlJmwHeAKCBFJKGbG0P4ZDO2fwZzxkUGiDzU0seKd2iOHpZZtruHx5duAoAN8UFYQGJFDUwX9eiZEB7gOMYmIRFFVe7APORMAAAi0SURBVIhlQViUb9nNG1v2cLAh0gHePyONCUP6BmdMZVOQ3YtePZLJSEshPTWpS4WHDjGJiJygvMz0qB3gy7bsPnJo6plVVe9ZL8kgIy3lSGAc/prRI4Veaf98ntEjmV5pka+RCwE7342YFBAiIm2QkpzEmEGZjBmUyVVnDQVgz4EGlm+toaq2jgP1jeyvb2J/XSP765qOvD5Q18i+ukaq99axv76RA3XBMvWNNLc4gJORlszUohxmjIrciKkzjHqrgBAROUlZvVKPdH6fKHenrrGZ2kMNlG+uYdHb1Ty/ppqngov/ivN6M2NULjNG5XFmSLd5VR+EiEgn4e5UVO3j+berWbSmmtc37KK+qZleaclMGZETBEYug7N7ddg21QchItIFmBnFp/Sh+JQ+fGb6cPbXNfLKup0seruKRWuqeWZVpHVRlNebGSOD1sWwbHqkxOY0XLUgRES6gMNDpC9aU8Xzb1fz2vp/ti6uOmsI3/jAmJN6X7UgRES6ODOjKK83RXm9+cz04RyoD1oXa6oZmBWbDm0FhIhIF9QrLYXzTz2F8089JWbb6Fwn3YqISKehgBARkagUECIiEpUCQkREolJAiIhIVAoIERGJSgEhIiJRKSBERCSqbjPUhplVA5va8RY5wI4OKicWVF/7qL72UX3t05nrG+ruUYek7TYB0V5mVtraeCSdgeprH9XXPqqvfTp7fa3RISYREYlKASEiIlEpIP7p7rALOA7V1z6qr31UX/t09vqiUh+EiIhEpRaEiIhEpYAQEZGoEiogzGyWma0xswozuzXK/B5m9mAw/zUzK4xjbQVmttDMVprZCjP7YpRlZpjZHjMrDx7/Fa/6WtSw0czeDLb/nnu8WsQdwT5cbmYT41TXqBb7pdzMas3s5qOWifv+M7N7zKzKzN5qMa2fmT1tZmuDr9mtrHt1sMxaM7s6jvX9yMxWBz+/v5hZ31bWPeZnIYb1fdvMtrb4OV7SyrrH/H2PYX0Ptqhto5mVt7JuzPdfu7l7QjyAZGAdMBxIA94Axhy1zOeBu4LnVwIPxrG+gcDE4Hkf4O0o9c0AHgt5P24Eco4x/xLgH4ABZwOvhfSz3k7kAqBQ9x9wLjAReKvFtB8CtwbPbwV+EGW9fsD64Gt28Dw7TvVdCKQEz38Qrb62fBZiWN+3ga+04TNwzN/3WNV31PyfAP8V1v5r7yORWhCTgQp3X+/u9cB8YM5Ry8wBfhc8fxg438wsHsW5+zZ3Xxo83wusAvLjse0ONgf4vUe8CvQ1s4FxruF8YJ27t+fK+g7h7i8Au46a3PJz9jvgQ1FWvQh42t13uftu4GlgVjzqc/en3L0xePkqMLijt9tWrey/tmjL73u7Hau+4G/Hx4AHOnq78ZJIAZEPbGnxupL3/gE+skzwC7IH6B+X6loIDm1NAF6LMvscM3vDzP5hZmPjWliEA0+ZWZmZXR9lflv2c6xdSeu/lGHvP4BT3H1b8Hw7EO2mwp1hPwJcS6RFGM3xPguxNDc4BHZPK4foOsP+mw686+5rW5kf5v5rk0QKiC7BzHoDjwA3u3vtUbOXEjlsMg74BfDXeNcHTHP3icDFwI1mdm4INbTKzNKA2cCfoszuDPvvX3jkWEOnPNfczL4BNAJ/bGWRsD4LvwRGAOOBbUQO43RGH+fYrYdO/bsEiRUQW4GCFq8HB9OiLmNmKUAWsDMu1UW2mUokHP7o7n8+er6717r7vuD534FUM8uJV33BdrcGX6uAvxBpyrfUlv0cSxcDS9393aNndIb9F3j38GG34GtVlGVC3Y9mdg1wKXBVEGLv0YbPQky4+7vu3uTuzcCvWtlu2PsvBbgMeLC1ZcLafycikQJiCVBsZsOC/zKvBBYctcwC4PDZIh8Bnmvtl6OjBccrfwOscveftrLMgMN9ImY2mcjPL54BlmFmfQ4/J9KZ+dZRiy0APhWczXQ2sKfF4ZR4aPW/trD3XwstP2dXA3+LssyTwIVmlh0cQrkwmBZzZjYL+HdgtrsfaGWZtnwWYlVfyz6tD7ey3bb8vsfS+4HV7l4ZbWaY+++EhN1LHs8HkTNs3iZydsM3gmm3EflFAEgncmiiAngdGB7H2qYROdSwHCgPHpcANwA3BMvMBVYQOSPjVWBKnPff8GDbbwR1HN6HLWs0YF6wj98ESuJYXwaRP/hZLaaFuv+IhNU2oIHIcfDriPRrPQusBZ4B+gXLlgC/brHutcFnsQL4dBzrqyBy/P7w5/DwmX2DgL8f67MQp/ruCz5by4n80R94dH3B6/f8vsejvmD6bw9/7losG/f9196HhtoQEZGoEukQk4iInAAFhIiIRKWAEBGRqBQQIiISlQJCRESiUkCIHIeZNR01UmyHjQxqZoUtRwIV6UxSwi5ApAs46O7jwy5CJN7UghA5ScF4/j8MxvR/3cyKgumFZvZcMJjcs2Y2JJh+SnB/hTeCx5TgrZLN7FcWuQ/IU2bWM1j+JovcH2S5mc0P6duUBKaAEDm+nkcdYrqixbw97n468L/A7cG0XwC/c/cziAx0d0cw/Q7geY8MFjiRyBW0AMXAPHcfC9QAlwfTbwUmBO9zQ6y+OZHW6EpqkeMws33u3jvK9I3ATHdfHwy0uN3d+5vZDiLDPzQE07e5e46ZVQOD3b2uxXsUErnvQ3Hw+mtAqrt/z8yeAPYRGXX2rx4MNCgSL2pBiLSPt/L8RNS1eN7EP/sGP0BkXKuJwJJghFCRuFFAiLTPFS2+vhI8f5nI6KEAVwEvBs+fBT4HYGbJZpbV2puaWRJQ4O4Lga8RGXr+Pa0YkVjSfyQix9fzqBvPP+Huh091zTaz5URaAR8Ppn0BuNfMvgpUA58Opn8RuNvMriPSUvgckZFAo0kG/hCEiAF3uHtNh31HIm2gPgiRkxT0QZS4+46waxGJBR1iEhGRqNSCEBGRqNSCEBGRqBQQIiISlQJCRESiUkCIiEhUCggREYnq/wOz15KSSlH9AgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_plot)\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Loss Plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ESRe0jKVxDQ3"
   },
   "outputs": [],
   "source": [
    "enc.save_weights('/ImageCap/models/encoder'+version+'/')\n",
    "dec.save_weights('/ImageCap/models/decoder'+version+'/')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Training.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}