Facial_Emotion_Recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0d07c521",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import cv2\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e15ca2bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset from: https://www.kaggle.com/msambare/fer2013"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "04547f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants:\n",
    "\n",
    "classes_num = 7\n",
    "classes = ['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise']\n",
    "Train_DIR = \"C:\\\\Users\\\\USER\\\\Desktop\\\\ML-Projects\\\\Face_Emotion_Recognition\\\\Data\\\\train\"\n",
    "Test_DIR = \"C:\\\\Users\\\\USER\\\\Desktop\\\\ML-Projects\\\\Face_Emotion_Recognition\\\\Data\\\\test\"\n",
    "Project_DIR = \"C:\\\\Users\\\\USER\\\\Desktop\\\\ML-Projects\\\\Face_Emotion_Recognition\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "035e7697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training_10118481.jpg\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABnYElEQVR4nO29bayt21Ue9sz9eexrJOcGQ/2V2kYmCEeqAcv94ZbSuglgpXWpCjWVwEmsGiRQg0QlbIhaRGXJSbFRpaqoRrbiVGBjyRAshBocKyiJxJchDtg4DjY45OIr34CdGvnec87+mP2x11jnWc96xpjz3fsc73W4e0hL77vejznHnHOMZzxzvHO9q/XecSM3ciNPX9m7bgVu5EZu5HrlBgRu5Eae5nIDAjdyI09zuQGBG7mRp7ncgMCN3MjTXG5A4EZu5GkuDwwEWmvf0lr7RGvtk621Nz2oem7kRm7katIexDqB1to+gH8N4K8CeAzAbwL4zt777933ym7kRm7kSvKgmMArAXyy9/4Hvfe7AN4L4LUPqK4buZEbuYIcPKBynw/g39L3xwD8x6kSBwf96OgIABDMpPe+/rjv90Oyslpri++5DmFdKp2r+9y9+p3HoKrrKjrcbwldKp2W6Mv3tNawt7e3VQeXp/ZaXZvpzPXwvm7jE3J2drbxCfn85z//J73352i9DwoEXO9ujHpr7Y0A3ggAR0dH+Jqv+Rr03nF6eorT01OcnZ2t9/n72dkZzs/PtwyTyt04Xhkbn4v9vb29jc7lwXSgNFN2dSyTzDGz+vl6Ngy+r/eO8/NznJ+fb10bbVfjiuv5Pi2fjVb3tf7s49oa/T/qIy1jb28P+/v72N/fX+9X9/NYcx/rdfv7+zg8PMTh4SGOjo5weHi4rofrOzs7w8nJydp2416+xvUd6xt1HR0drT/Hx8d4xjOegePjY9y6dQvPeMYzcHBwsPE5PT3FF77whY1PjNv73ve+f+P64UFNBx4D8EL6/gIAn+ELeu/v6L2/ovf+ioODA2tUIe57haSXkcygs7oqUSO6KoO5SrtmnEivnQHQy9T3IMsasbfLsL7q/GXH1NnGVa6/im2EPCgm8JsAXtpaezGAPwbwOgD/Q3XD3t4ezs/PrTPGNq4JydBbI+TIANTxGakBrOuMLVO8rF73fVY0qi+5XuuuANTV03tP+7VyiOqcfr/fU4HQV9vFTIOvyfprZgqhjCpsdm9vO55qXaM+HbFOHtNsHF39Ixt6ICDQez9trX0/gH8EYB/Au3rvH6vucc7novLe3l6ZH3DHsimCm1fpMS2PQeiylD+LwNrmrOyZ6OGML8p1Bqv3V+A240gKQqMyryozEXsJuLoxUgA4OztbTx+r6Vmmz0xdVQCrgGAG0EIeFBNA7/2XAPzSzLUc5Ue0XOeLiphLJermXADXq4ZbOefoeIXYI6nAbXSf9lkGEPHdtXPJ9MuBwf2WyqG17vPzc5vIq8rVPtLvCgRLAMCVp/Y8ypk4n8jacC1M4DLiovCICXDHKVXP6lBqGM4fSRmuD8AWMFVR/H73xVXrGDmzUn+9N7tvRJfZCTMmsFTvTJxeGQXPpgyuTG2DlsMAEInqjO5rIHGs4X4DgeufTHYKBEY0tXIOZ8zOYdkINBur9Wcd71B/NlJXhqrHXeSu9JsRZ9wZEGT3ZrpU0dPpfD8BdBY0ltQ9iujBBkaOmt0/wxiW6Lu0HSE7BQK8BZatDXDzPi1fz/Fjm3hCMVPnDOBU9/J25ISj6yvwWcJeljoRl5MBS1bXg5oiOMmcMOvzjCm5PmQAyMBAx2JWHAvg/q4YwNL+3TkQCFlCHWfmY3wuOo8BYH9/H6219WCenZ2VlCxzSt2vdMiMrKLwS6I2fyqQmQW0rAw3BXAAsaTMGXGBg89VDjijj5bv2FvFBly9GVty04bYjlhGxipnZSdAoGqQa3wWBZ1DZejJuYAAgpCZ/EJWp5t3uv1ZOu4QPyLPjGRG5q7Jjs04dNYHatgVg5mVJRHV9d1lysqAJPu4OpayrdEYj9o2KzsBAplUTl4BAR/XaMCr4jgXEPmAzMFGdC6L/pkBjABAQUsj+ogRjCLFDFtRXZcaWTXvvV9SgVXsZ85SUX13v9qWmwYsYaaZKFiOpnyZ7rOyEyAQncnLgQFvyM6pteExsGoM/ChQHwnygiAHIAA2nkxkKJ2BT9V2lsz5ZyJ65QSOaur+VZ0/mxaMAOsyMuNcru+WlJ31o2N/asNLp2wj/Wb7LwtClewECABYz8HVCdkB43FdnNMVhMA2Hef79XEgD2LU64BIFyjxfDCTCgQu6xDqTJdxrooJVJF7NsKNIqvKVQBilgrPsIDKebJ9/s7rBrJyZtpyFSCYZaEqOwECvfctEKiieSz+YFBQ0SiqOQB17Kg/9IkyYqsUkK+bqT+un4mMIybgorv256wxVdOCSkaA4ah2lktYIhlb4bJ671vrTRTEnP6uPQ7MFTyYCZydnW0EHp1CcLkjlnhZcfZSyU6AAICtuZXrJEU2ZQOOBbgcQIAAozevA9f6eF8NzbEB1X9pZGT9dUCzPElWlgMh1mHWCC87172feYElhs338LX8S8iZ6ys9WAIEFHhmorGrc2kbZ8t1shMgoHQc2HRiHTjuYIfKAOz8X/MAWQLGDZxz5CziOQBigMt+fzACAHeey+H2V4YxuobLWpITcGW4dmb68DF33cyxkWT1ZU6YBSL9zmPLSebLgGBVX8gol8QL3x4aJqDOoA3heTgjLd/DW/09OQ9WzNsUdFzHZXRQ8xPuPs098G/ys4iUOXsWUbicEeV0/akyO13ZBXHg667JQMY5uPtBmavTsSwG99GPirIonYG+6sDbURkPJQgA20DAzp8JG7uuBlS24X4WzAOq5YZxhBFobsJdz0lILVMTVKqDGhoDn1vePIoMrTXbh0vm6ffT+WcYyf0obzSNcf2kv2YdXe8CjNY308YR+Ltxc8EzKyOTnQQBjuZZ9r8aTHbAw8NDHBwc4Pz8HKenp+scQCRwnJNl5fI1jgmwTi4JyW2IN9CM2qLtckatUylXRhxjUHV9nx3TvvlSsYEZ0M+YSzVdc2VrX2tZbHvM8EJ0KqvTgZlkpLOpzJFdmyuQymRnQCAkGyBHw6qoyVGYlwRHWfwyCNdZDmFdRHUdnD2JiHLVODSKZVFJHZjbpKzEgYf2lxqotjsbmxm5HyAxKqOaDigAaPCYsSUui+vQ8VBgXpJrqtpdXe/Kd/fPjMOlXy/WWntha+2ftNY+3lr7WGvtb6+O/2hr7Y9bax9ZfV4zU94okaeDrY7u3o8X1+ogVslCHXxNjlVzPLcWQdclVNHGGSIzGv4cHBzY+vSdd6yHAgoDwNIE4GykmTX87L5MZnTVRGw2/XBOq32SBZolETcTZ9ujMnl6O+qLkW5XYQKnAH6w9/7brbUvA/BbrbUPrs79RO/9x5cUlv2MOGMA1c+OucERtXkww1miXu6kKmFWAQHrpUyAmYjmEHRwuDyX2wCA09PTLcPRdioosu4KjCPJom12zZdSRu1w7c6ivJtS8daBaPQ9i9qTAx837hUbYR2zz2yfqFwaBHrvjwN4fLX/Z621j+PiVeOLpXJqRfO4HsCWgWf36kBEXW5QHM3KDEj1ZxDgyBz1RY5DBzkzCMcE+LxOLVhnjVLu5SgKDpkuWV9xP7s+4f3LgM9VROvJxo7Pjyi8C0SR3+Eppusf13/VuGu9TtdM56VyX3ICrbUXAfg6AL8O4FUAvr+19t0APowLtvD5URkVCGSvh3JG7OZmCiAuqaP3xnd1Li6Xy1QW4HISmoxUcUa2t7eHg4ODDRDgfhm9fl2jkntS4drM/czlOflSs4MRKLt8AINl6Kb94PIq2TTOMQHNvbAuIc6GK1vQdo1YwGXkyq8cb609C8D7AfxA7/0LAH4SwFcBeDkumMLbkvve2Fr7cGvtw0FvXUfzfNrNdTUn4NBU6rXX83m+zpXj9HTz8Srf4PTS6B9TgPhELkDfhOT6bGn9lWj/jPrY1TXDKEbH76dU7DH7IdDIFmb6Je7R8irGUbUhm6YukSsxgdbaIS4A4Kd77z+3UuSzdP6nAPyiu7f3/g4A7wCARx55pNM9Wx+lZhzhs4ZnAGD0SJNAeh8fY+aSJQSrQXR1VQAQx4DNnEDmoFzHZeb/2ba6pzqeTQf0WAYS2TUz+riImtlCSLUIq6pvFOH5ust+nOi0YAlLuDQItAtt3gng4733t9Px567yBQDwbQA+OlHWVuexE2lj3Mq7uJdpn5brBt91mMsZ8PVxnHWcYSZcBtet7Y5y2PkDDILSc9/wfdpnbt44MpAKAKrpQlWW3ls59QyAL617ll0ocLrp00i/mejubH4pC5iV0bVXYQKvAvBdAH63tfaR1bEfBvCdrbWXA+gAPg3ge5YUygatAxAdo7/WyiK5G4wsCmTP17V8HexwSpcDcAPpdOV9BgDHBHrv9qWofG/0EYPlTL87xxxF+SxSZk7rWF1W7v0QB8Da53wuyxOofjM6j5xc9XM2swQULpsjuMrTgX8O2P8cnPqvAScaDavn6KRHuaqwGhgGEr0mHIqnHlp2iJuLq54q6nBZbiEAILaagda28hOTpY6UgWd1fXadAz8+7pywKnPJdCCuya7LGKHuZ1MXB+ROtwoEsuNL7gkdndMvAYKdWTEYjuOe2zs2UEXrmcikzhPn3QIivV8HJHskGPVou+IeBRjNB8QfXmYLeyogUeDSvuB+02sc28mkckh3Th3Qje2o3JEs1Sk7rnrG/wwwGLh7tQ1LAICBfAaUr8IeQnYGBEaKV4YM+Lf8jDpCgQTYXrkYzsTRVQfLPRVQZ+V7FGDisZSbCjhQ0Xm+9omrI+u/qNeBCu+7+2eMrAICN726HyDA91fOnemvujGgtra5OCj6MHNE3s/qrQBh5pPVWbWRZSdAoEI/Fhc1FAgqdM7mvNmjtmrQNPq7/y/gH/aocypQKJgEE+C+cUk+1ck9mXDXsV4VsGj/jYyqAlwWR8kfBAhU59XWFJyYLbmffgcgVNO/KppnQHEVIOA6ZvtyJ0AAqDPyLBl91PPOgbOIBsAuI2aU5/vZ4XTeHiDAbyyKcrO1CVG2YwLchtF0h4GJ35mQUVfHWNzcWPuKdcoM8DKSOctl8htL6nPjq/3gXmzDIMr3atkKynpNtc5lxvmzccjGSGVnQGBpFFD0ju/uiYLe54AkyxHodSMU1kidzd9d/S45yG10ALBksF3dTs8RGGjdWfkzeizNCczke/TaWadx0xMtz/WR7mf1jNpX6ZiBiybPszKrvM5OgEA0iL+HZMq7wcg6Vu+bOcZ1c6cDm79Z0J8n67n4nJycrN9n4PRt7d5ag2xOrw6jP1DKnJjvqQw9e+mKM6LM+SuH1Ijr9pcEgkofp1MF2u7ekQ6se2w1d6BJ3czO1I4rcGfGF3Y5o2smOwECQP7bgWzgKhDIDFSTPW4QZ4yCy2DH4WtYN36RiXulOYOMAgEAe48mMJ0za/muT3jL05cox/WV6xvto+waxzAcS3JApeNX6TFzXHVxjKFqi5alTs6PmDPgzezYtSGzlRnAqmRnQKCKxNn5uGYJE8jKz85lTuQMWMtzQMBR1lE793TCMaPMEBxFrfpGP5xLqBiZM2z+jADDzaPddXqsGjMHbFlZLNlboqr7RoDiplQzNqx1KCg5e8lsNNNVZSdBoEJENXJG2goMquNcXmaYbNxOB7eMmcvmqUEW1bPfHcS+exNzTAXceon4njmeo5Pcntkx4eOzjjfr0E4cM3HOWzmGAzWtf6Sju1enOY4JsL4ZA80c2wWKCqxn2rAzIAB4B2fnYerN26yjWdwguKg5Kod1dYhfRTWmnMD2ewh1iXD2jsXz8/P17whC9NHjiMHwfQoCnPXm65SFOCah07qsbzInHIGIArErO75rHzhn4PF2dc847yjCZ7pWLCgDMgfyme6zsjMgEI1xjh7zaXW8LOq5jsnQMKNilY4OrNixWZRZcH3Z0mAGAufMXIZOGYIxqBO4aBXXc/4h9t0aAmUD7PhaDo/HTL/O0nEeY9WJ2+RAwJWh9pI5faX3CAhim9lZpm8FRuEb/ATpsrIzIABsUlGePysTcGxAEbqKyFndo2v4OofyVZImjIWTRRz9+cUhzAi0PDY6jvysk3NQB0KsNychOR/h+k8dlQHALWzSvqukcl6nfxVJlZk4x5+VKjI7IMiYIt+TtVHbUPXDrN1WsjMgwIYcDs8MIJsCVJTcRZgZHdw9WTTXOrOFIcxkwsF0daBOBxyrUIOLqM19o3rwfcpc4lxEFPcKtMzgHCPIVje6/nS/bZiJ4nyvjp0rw42hE9cu/u7aoOOR6at9kdW/xPn/3IIAOz8/V+ePPs92KFwt+62ieeXgFZ3j5F5mNBpl3QrBajqgKwHDcVvbfPU4t1nZgjPu7KOr4SrqrUnGjAYrc9IEpIKJijoU61QBwQwDcI43csqsH5fUEcfd96osBvTLMhxgR0DAObabAjAIBFg4Cswd6KITA05GLTMgALb/bdZl97V9PBUIR9XXkmevC+c+UMc8PT3dYkjcdgYK3VZGrXN7djy9T9ugfe3ocOjMTzUcmGr/Zy+UcW1TNqQ/BHPtnvlwXe7emSXAo2sqyQKVu9ddq7ITIABgHeEiF8A5AQcCvPgG2H7LD7ANAuz06jQsGboquGgdzngVrTmZ4xYHucVCGsnjPQgKhApobl4foKFGo+CRgYD2UeiX5QRGfax/5Z39JDu7X8U5W4jmTypAyMqr6nSOrcfcs/2q/IpJuX4dAYiTq75j8NMA/gzAGYDT3vsrWmuPAvhZAC/CxZuFvqMP3ja8lAkEADAIcER20dBNGSqUdAyBncc9r3WP22LrQM29kkyjqpOTkxMA2AABJwxU2m42UJWKCXBfu+jNTGAEtFxO6KuPTVkn1SFja84BZ2lyFvUz0NTzlYPrlMnVwe1w/TXSXcuakfvBBP7z3vuf0Pc3AfhQ7/2trbU3rb7/0KgQZgL6ZEApPH/ipZsuWjMzYBYwAoBKGFj0rb5ZBHXTj2AE1TQgo/J37txZt/n09HSrvfpdjZeBgHXl+6oEX+zztfo7Bv1NhbZDwVunVfFrTDVm1omnA1W7M4dwLIC/OwfmflJGpVHe9Sdv3fkZ550BwCXyIKYDrwXwTav9dwP4FQxAICKlRnmN9Dz4+iMXYJua88C5aBiiA69bHWhN4DmKp+zF0Tx9GqALhXhqEVt+u40KG5fSXmAz0obTsf4c2d3j2Aw0VWfWJ/TlY2yoWq++l4HbryAeerpxmnEKZQhuzF3U5v7Noj2fcwAwAmytrzqv26VyVRDoAH65tdYB/N/94jXiX9lXbxvuvT/eWvsKd2Nr7Y0A3ggAh4eH67nqbE6AjZKdX51yrWhixG7Q+DsPWJTrVvbF9bFVg1XGECCg/y+o6wWcvmxMUZa2m8FSnUzLyvrKTdFo/NbbrC/4fu1j7heeDvDTkkovR7uzMUzsb2usMkerorTaRgYC2VSgcnwuI2vLzLmRXBUEXtV7/8zK0T/YWvtXszd2+t+BZz7zmZ2z3AwCmfOrQWoUyea01VzOIXQ2V2c2oIgt7QRwL4Kr44azOzAIEGDdMyPiKUQ4oDqQJtrUcDUy6tQrpmzad8oEVF+uT0GAgYBB4PDwcEtfHvvsr+VnAEBlJiKrLtoPMyDg+juuyxjHSEdtp2MHo364Egj03j+z2j7RWvt5AK8E8Nm2+u+B1tpzATwxUc56bjvzW4G4x81jswSdA4LqPt66R3m6qo/LdbK/v7/OXzAI8EIhBwLKgpwhRZkMHPpmITZkB3LOWHh65vptBgQ0d8AgwI9MtRwF8iiPcyk8leDyQ2ZBwLWpAhYNJMrMKhCYqccB0IxeDgBm+uEqfz7yCIC9fvFnpI8A+GsAfgzABwC8HsBbV9tfGJUVxgbcS55VKwUTfexAcJlMOwFsRM8MANjxOR8QbwLmdwpqDkB1iPKibQoo+pqyg4MDuzxajaOaSzu9HAg643Lt4PtmQSDu0ekKAwGDIk8H1E6CBWgewznGjPC1oynGyO4UaJeCQJY7GTn/qH1VcAKuxgS+EsDPr5Q4APAzvff/t7X2mwDe11p7A4A/AvDtM4XxAhDHAjSah3AnaLR39/C8taJpPHicAAwA0Hm76qmDFRFsb29vnf9QgOFoHuXHtKi1zR8GZQAQNNo5rvZnFnHUwRx9rfqIx4Prj/MKmgEELkHq9OfEMdtA5qzRDi5HJQNEV64ys+z6ymmr81zXTJmq08jpVa7y5yN/AOA/Msf/FMCrF5a1xQRcDoCNF8hRkAGF79HHb2r4rpM1yrGDHh0drSMvMxeXuOLotb+/v/GdDVDrCObAwKHOGk7I9zgQCr14WqUGx8xoxoAdW4oxVVDUnIiCtvazJlyjHbysnEWdQmUmai4BAXcPlzUbtTPHzs5X7VDgHV0P7NCKQRfxMybgaKpjAMwq2EjDWWYGnaNcGOfR0dH6E+Xo+gYtW52e2UjUqdODo6OjtaFX82sFqAC4rC84GmrkYId2zh9brZ/vc0xOwUKd1E2P4p6oM4LFwcEBTk9P7dOTKEsj/sgRou2j6J1NqWY+Tio2sLS8aqpcyU6AQBgL71eAEJ3ArIDvjXNs+Ox84cBVp8Z3jU7haAwEADYoqjprRHMGAc60hzgmAGzmSdgB3VTg6OhoDTBsFKEb951+2FE5t1B9+B6O9Ay+OvVh/XRakDEBBYEAAgaz2CoAcBk8ttpHVVBQu4j7XA4qc2aVWefOzldBkK95kDmB+yrZXD6b6+j1/NhIr2GD5Sg7AwLsFLx1WeCog5OAEVFbaxvOrBIApX9EGmXptQoYR0dHOD4+xvHx8dZjS+4HtxIz9jnfEf1zenq6zmO4tjoQcCCu/ejGs1p/Ee0JEIh8CTtlFjxiHHWsFUC4XTq2mb3OJv3YpnQsXR2Xmc44qVhIyM6AAJAv6NG5I7CNgvwcW6cL7Pz8SC6jf1E+O1s273V/8qFOwqyCo7HWE3pxPWq0bHwOAG7durVBk5kdueXYATLhhIeHhzg+PsbR0RH29vZwcnKy0V51FNc/mh8JEHAJP9ZTn5K4KUno6HIc2Tb01vFQoKicN3OkUU7JMacRWIQ++lQr8w/uQ5ZZ0NgZENCoUEV/3udoxuc0OmmUjQSaJuZClAlEpOZ5qtbpqKXOa+O4GiY7iQMCF9l0anLr1i084xnP2Ii0/HQhpiUKAtEOZgG3bt3aMNoM4HSuHzrzAiPHBNz4OyagzqgglrWJE83RXzoezs5GDsq24Zjg6Jz2nSs/Y8UsSxjESHYGBAD/azXtBDffYRDgqMN5AJ3TMwjMoLib+2XUXoGF58Val87rFQi0vnDEuMcxgaiPV14GE1BHYkOKaVKUw8CoTxR0GjACAW1flMOSMQHVkR0+dNG2ql25xCSD4IzzZk4+e98ICJytV1PiWRkxgp0CgRDX2JkOYCdhg9VHg247S+cc8EQ9sR2ByAgANMnGNJFBIJ5McJIyEoOnp6c4PDzEycnJ2uiZhehvNDgSM0gCsKChLEB/TBVlx9uhNOnHfcbTkYwJxLXn5/fetMx9EYnX8/PzrUeHMW3QnAS3x00zmQkxIGbMICRLLmbRO4vmGQO437JzIFBRIaVsvJ85nUaoEI0YjrZnAxkSxsPHeY7ITqzHIzpqNtwlHbVvgukAF3mGmMNzrsM5HfclP2sPIHCP6M7OzrbKUfBUx42yI3mnTCASl3F/XKNLpzUnEu1lx2DAifbwPXGtYyIMbgzoPL4OwPl8ds/MR8d1xHozELmq7BwIANuJET7GsgQAmFprR2dI74CGJZw5rtGowftcj4ukClY69eA28zW99w0A0DxIloiLiLm3d5H8Y2dXpnRwcLCxQs/VoSAW4BL0XUHAzdGD3bhlwwG43BZmAQEAUbfakurZ2nailK93ORvWx42L7o8AQIFgFgAyQLgsQOwkCADbi4ecOJR2A6eOFVTQzXHju4v+WQer42e5BjUmpajV1ETrArCOzMfHx+sI6pKYXF/cz2snolyl9fH99PR0gwkoUOhcPhwyFvQEwPB0I/pjf//eC180WqvTucRrTH2YVcQYx7THMZHoD/fIlKca3M5ZB3PjpTakTCX2MyDQsXf1ZDqMZCdBYJQMqTK51Vw1rpkFgcrp9bsDAOfMGvmyObWrn6cD/OFcAIMARz9OjoXxMyiGPspIlAkwwOj0gRfxxGpHzgno41l1voq5xZZBjHMBupQ4W6wUOsQ17snCLAjENSOJa7Jx1TbOsl5ni+6+US5tp0CgQjY+7mhURpWqTuWto1+ZHppEcgCk7IDPs6NWi5Ci7Ii2EUGVjgcDYMfh8nXpbjhgRMoACTcl0WmFAwF2rrjm5OSknJ7Ed03Q8bkQpuo81tm4R7v0HQ4MWMyGNDfANsP6sENVjpVFcDcNjfPZPQr4FZBcVnYGBHRQ41gl6uAZAOjgZpFenyxwuZU+GSPRtuhAahRVENBoFNG+MmzWaW/vYgFStFtBwD0dcEbHDjxiAnGNJgsVMPb2tt/8q/2jtqFUHcAGgMQnph8KwFo/swkHBA64K4bq7FLpvnsikdmaG2Od4rm6l8rOgADLyEn5ewUEeo1OI7Rs3sZ1lY5Vsi9jJjodcDkB1YkjehaluVymv8EUgM0nF/yIUJ8OZAAwAoGoS1mOluHm+wqecT6chum6AoT2b0wLHAthZ4ryIi+h0wN1xBCdOjhbUl0VDJyNZQHDMbSRLAGDnQQBIM90Vg5cnVea54CCl9C6enlf6ZzLCWQgEAalKxiVxXDZ/Jgvo4eqYzhlGI0C0fn5+dZjPKW/PO+OYzxP1ggfeioTcAbN/er6Ko7rfxPwfeH0cU+sI9D8igJBOD6PPa814B9r8ZjFuIeDO4dmEHGJv0zcVDI+7hHyTCBT23VylTcL/WVc/L9AyEsA/C8Ang3gfwTw71bHf7j3/ksT5a23zqizJEzFAuK8YwtOlGqqftVxBwBMa+N7OBuwnYjLmE8YQRiVGrj2A4ANw1UQYNqsj/FUHz0+AgFuWxbNuCzn9FmbdDrHfcSrISvgYT3ZNiIAKKg4EOC6ND+kYxdla3uyHADvZwzK2bmWO3J8lqu8VOQTAF6+qnAfwB8D+HkAfxPAT/Tef3y2rNbaeu7KFHVVzxSdJ722ynbAULRry8C0PN2ysWSGDWxHMabFLtnIIKDt4fqyPory2Vi1rrguW7IcUS2AIjNOdjp2KAeKFVhWY8m/A2DGFE7Jv1NgoOJ5NT9JYVqvS6i5D5m5RPkuicjCr9B3jyC5/7lPtF598qI24+xT7W4k92s68GoAn+q9/5slCBSyt3cvgaVr8fl7hZ7uPF+jFEodJ3MS3bpO1vLdJ9rJ3zkaKlPJAIDLia3OUfkabqsujGGHDkfXBB/fy+sEso86v3N4/q4siPtBV/Fx+9lp+MUrLp/BUx1eT6HJQAUABkgGRZ4yKAWP47Efi7JcXTpVifYwaClAV+yvYgiV3C8QeB2A99D372+tfTeADwP4wT74G7LWLta/60CwAcZ1o/m6UkbnoFeRytGrehQMYqtMQNvEQMDt0rqynyjHh42Ty+GIqJQ5+pvX6zPDUHpd5Sqy+S6Dh4KyAqBjGDru7hp2qmADHM25b7TvdC7OeQPWWQE4dOKXzXAS1jm/LqJyTCADAReYYjtKEl4ZBFprRwD+awBvXh36SQD/G4C+2r4NwN8y9238+UgYGQDbsQwQSyiPi8izDKAqxzniDBtw5UXdLhLxvgM31p3bwgYWDsROrY4IbP+Kj5kAg6vLB7g5q0vOZXmCaItjNdw/mo8I4cReVg+/IHZ/f3/DMbnfeZ8dMerXJKJOKZiZBBPgVZScaOUnPjzdUeByj4EdY1VbGwEAcH+YwLcC+O3e+2dXlX42TrTWfgrAL7qbOv35yCOPPNJ5UYsDgdU9Ue5WY52D6pxQ6TiX6wyO61SpOj5zfj3mQC2rR+/XebSLotz+iEDqlEGXI+px5GPQWAICvEJQy3QJO35kx2DHjs8Lnris0C++s4PFU5eYbvL/O2TTE+1fpuTMqJh98bSFgZrHhXNd8fgyxpTbHrq4ftNyR8LBo5L7AQLfCZoKtNUfj6y+fhuAj44KYMNyc1dNEsY96kQOOXmp6ugxC5etQKDMQbdxXxzjctlhKyDQcrSP9P7MKHq/t/CHnZnnwRxFnXOFMWYgoNcqCER/Hx8fr+t2SS6uK/qK592qgzIBR/8ZAOJe/X0Fz/HV+bnfdOpRMT0HIjr2amtcpgIAM7KryOj+q/41+TMB/FUA30OH/15r7eW4mA58Ws6lwqjO2dQQlx+IrQ6eAsDotV1cftVhIxCo6Jlzfuf4I0agBsbHtD85wnHSLPqay441A27VYCQEgc23EWfJwAAh/o/JKIfHR9lK9KEDHwYBdUynL1PrAAH9uXW0WwEkWJMbQ9aPbY9toAICtRkeAwUwHcORaJCM7QNlAr33JwH8RTn2XUvLieiiA88fNVy9v5pHVSxAO9eBAR/LQEDZA+tWgUFco9e7MkblMbXk/gjh1XFMR4NCO9qu7RyBQOgZABAsjseGI7k6NBtyBgI83qw/t5mTbL33LRCIcxFwGAA4WcrjwwDAfePYBG8dY3NMgFkWA0HFDCvJbFxlZ1YMhhFookXnuRUQaEZV/9vPGWzlkCrq8KEvRwbVafTJJBv4rAxlAuooACwAMPXO5u1aJjtzBqq8ElFBWiM3389OEv3qQEDb7lgO6x/TAbYDbjuDgVsDEDrMjKkCQsUEGNw1gaksaUYyllHJToDA3t7Fz2GVWjMgOAeLraKn/qmnW0Y666AzSDojswAwA0IVG3H18Xc3v43vzrm1LJcM5Dpiy7TcOYWL6NpOzUUw0GqZoTvnLTivFCzATQe5D7hsBYIZh9f8ysHBvfcpcN16jctfubUBI7mMre4ECLTWNkCAM778/HtkAAwCPOgu+qhBZgBxv9p32eNZ4lHPVWWosbNzR6SMKRf3Jd+v0ZwpvermnDLu563ra3Z4ZgI6TdM2MYXXnAKADRbg7InL4keonPl39scA4vIL2dSJ+4/tVv/n0k0H1CZ0qqp28EBzAvdLWms4Pj7emgrwG2McE4hzMwCgEUANsTp3GWBwBnbZskJ0MMPotE73XUEz6082WOBeYo2dUGk96xflMjjwfgbo2kaXF3KMR6cBYfR6j3sSwToowOh0oWIDfE/GBNSWs+ClIKDBKRMHBGqDmewECMR0QFdwZZl8YNtoeTWYeySoEcBFpsr5LyPZIFwWAByqV1MB1YGN1DmiRqgonyOyMgLWia/h/ua8gtPVtacCAZUKCDk/oLkH3nLUZzbA/eM+ChaOBbjHoQoCPBVg+1UbdONf9eeM7AQItNbW/yrDv2yrMvlquIqqjoZlg1gBgNan35d0+hJgGVE4vo4NpDKIkTGzgzvDY2rPTh7sTesJ5xwltmbbmkkFAnrc9UnoUAFpFv11y30YORG254ODzXc28p/HuER21aasjUtlJ0AAuPdjGjePUvqaGbGibDXfH+UCZqLA6BP3u63uVzIDPsoUdF/v530FQi4z04PPVxHbtVETwO7YKPqrXgw6s/o70TqzMjQPwFMzzk/EUxJ2bA5W/McxLm9xP+ShyQkwDa2iuN6XObUCQTYduB8gkCUUs6hTXVP1EUvm9Bq9+Rp3bQUATjeNnOqwusiL2YLTm/ViB3SgoMedXjF1yUTrdpKBmfYxjyPXGywgyog1E+H88Z8MwQACBI6OjtInAk7nbIyWys6AQFCnjAVk0VCd2AGBSwo6x3UJo1kAcPNs1tfprueuIg4U3PGQGeDSsjKdHRNw91SOrI4+AgYnWteMw4/a4ergevhJBHAv99B7Xz9hiWXUnCismADbpwPuTK+sPSPZCRAA7hmlS6hUTwgqJuAo/0wOoAKBmScHsxH/fgMByygaVlu+P9PNRWp+Ll/dG+eWsoFKMidYCgROH9WZoz87f+iuidAAgfhbOH2SxTmBAAFu01VzJiPZKRCIDtSsaZZdHUXoGRAA/Dw5ZAYARklFLqsSh/zVtZcRNWTn+JU+Ti+NnJneLsK7KcQSGQHG/XYg7Xd2/uzDjJRzBPpIm3/TsIQFOB3589DkBGLLj05iLuX+YqsCAs10ZyCgdet2BDKja5a2nWUWAKpoPiM6r1QH0rpmo7cDCnctA0BWdqW7K7O617HJaGcVebN+5T7JQE6npu5xoK5mrOb7FZBn11eyEyAA3GsYg0AsGMrYQBb9MyBwzht1sx6xzYBFj7vr+dhlxVHykc4zUgHO0shZzaE5P5A5SHbfTL2ZHtk1WVTU461tviPAXc/3ORDlLdt02LUDgrDtJf3gQG2mzSw7BQKxjUcsjglEovAyU4IMBDI9qqhfJdau6vwqGqkyfflYda0rn7dcZ6UT72cOzef5vnCy7J5Kqroz5x1F11kZRWGO/hkLYBBwS4TZcSvmMtJz1hZ3BgRCosOAi4bE45T4xF9PZ3P+zEkv+xhvySf0z865cvnYyBkcPedzcYwjsMoSJ5ih4rE/iuiOMi/Ro5oy6A99VGb6wYE8v5ItK0fLYBDgnyhrXsD9yI3tlNu4JLoro5nRe/gbxdbau1prT7TWPkrHHm2tfbC19vur7V+gc29urX2ytfaJ1to3j8pn5aMB0WH6COXWrVtbvwt3ycJsvm/aZr+PInx2bCZPMMsotHzuJ+4vFwmzdzJk5Wp7dN+NlepQ9fMIAGaBSdtZ5RRGgULr5/HIAote7+7PPgoAar/VWC/pJzfWI5n5ofLfB/AtcuxNAD7Ue38pgA+tvqO19rW4ePPwy1b3/F/t4j8JhorHYMZAuOeoAQQOBGaidEjlaDPlzDpztn5hBjDUYDOnH32yzHumt2tfNmYzUd05qJOszqpN7nf/WlYGAFmbs8fLl7EVLVOTgqNH4NqHWX9pX2XXZjKcDvTe/2lr7UVy+LUAvmm1/24AvwLgh1bH39t7vwPgD1trnwTwSgC/OlEPgO136gPAyckJjo+PcXJyYiNd9ugwOoA7Qo+zzA6qlquPG5dMTbjuql/0WGueEnJUjo+bRri6+bosulfsIrZ6ToFAdczaOmI9FavQvs7K5vqZwsc5ftlIxXpGwMC/ctzb29v6fYvaFY+xjlU17pdhApfNCXxlX71MtPf+eGvtK1bHnw/g1+i6x1bHhsKK8zwKuACBk5MT3Lp1C733rT9zcEhaOTBfk+1n97lIwOcyh88SmVp3fK+cSI3DXat9G9dnfeH0GB2Psp2xZnrpsczAFUwcE1CHzADAPct3wAdsAgHrykxV25TZCOvB54LpKhMY9Q+XkfWv3puBBsv9Tgw6a7EaNPrfgWc961kbfwC5Or9BoY6OjnBycrJ+n1z8Rts5HtWx3s6CwYbipvM4UoRBss48cM7JRxGEjVSdwFH7LMqqzqxf1e6KAcRWI3FmrJk+ru1VHW7+r3WooykQ8LUObFSvYAGuHRU7c+WoTffe0ycCldNW/VzJ6J7LgsBn2+rV4q215wJ4YnX8MQAvpOteAOAzroBO/zvwnOc8pzMIKMIGasbrq/f29nBycgIAayCoolv2qTrbHXPRO/TNomEV7R2jYL3Y0fg1XVpO1Y7orywSuv7SfsgisnOkqlwX+V2d7Pw6/1e9tJyKcVUAULEYd9xFbHcu6o/pQGzdD4UyG3rQMv8Gw035AIDXr/ZfD+AX6PjrWmvHrbUXA3gpgN8YFcaDrG9xiUGNjCo/HXBvi3FSAYE6lhq5RqKZ7Lur320z3VwScZQ8Ct1dO1jvUR9VkrV3FCWztmpfsLg3DWf1Z32Y1aVtzQDa2YbrB6cjlxNj516AOxpTB04je1sqQybQWnsPLpKAX95aewzA/wrgrQDe11p7A4A/AvDtKwU/1lp7H4DfA3AK4Pt672e2YJFgAvETzFXdG4ssjo6O1tdrdMiipA68iw5cZhZZq3NxfCbSOv1GTpGxANXffWfdY38GsLi9mQNqFHV9lEXYEQtQx8rWAqgzjxaG6bg7favxq5zQMacMnAIQWMfKxiq5KiDMPB34zuTUq5Pr3wLgLUuU4E7jqMUOzvkBfXtrXFNR0VH0d/v6fVS+cxJtGyeeqncEZuAVxxhsnPFoO9wrtEdtcsyi6qcZA87YEN/npgEVeFXjOxvJtY0OkJb2IZ/TR476k2Huh5kAcj9lZ1YMcsfyS0YBrP8RxqFr9gy3kowJsFTU2ZWXHY/2xGBnQJA92uSy4nqVOJ6xm9hmdHrUDpalLKmqI2M2Gv1dvUq1s+mSlqUA46g838P/JMx/XKrBoXJkDmK8enBmiqf1OVDjvl/iByE7AwIANjqek3+aKwC2G1tFehY3OJoFzhzF0eqKKbBB6Gus2YAVKPijOuijKz3uMtrKRDK9ZyJbJktBxTGBLPJyHUqrR4lAV6bL8SgQ8PEAgqyNvd97eWt2jXtbllsvov3pQK3qzwwEK9kpEADu/cW00lxujGMCVYcCmw6shpJR3+oR0ahj2WD5w/N7jWJsTNw21lmd1zmJ09MxAhXtE9furO7qvBpoNUYjxpJFfwYG1ScDgIjwI3CIa7P2MKNzTId11TddZf3A+7OgPOrfTHYGBNx0wNEqlhETGEUiBYYl1Fn1zvRR3fh7PC7SDwOAa7PqG8JswIEWg9tM34yEAVr7QvWdPa6ROLvXgQADZcUsXJR3ju/yU1G/A4AK9JgN8B+7aNuz4w8SCHYKBCJSsmQO7RwryoltZpSuozj/wPe7SJQBDbMHNUp3fUQXnQo4A1fdVadKPzfVUHERnx3IsbHK+ZVhjaKeu87tcxTlaKpg5Oi9o/rVR9vsbEHZquo7E5xmALcSFyyjf+Jn+ZXsFAiE8KIQHmxdC5B16Mxz/MyZNEcwcjAFAXbqUXvVSfSTPepyTqDRO7s+S54xAGeMiAEumyaMRMGa6+FreBEWt8e1HYB13AzE3PSgunYkcQ3nemZAICt7CXi6axgA+K3HmewMCADbxsfGwD/AmClnBACuHE7ajZzfRVauSw1Cjd5F0sxwdA6cRUMtn/sv2qd6qw6u70ZAOgOwmWRszT31yZyJqf0o489tzCJ/Biiz7Yjr3eNf7nfHILI+zMDTHXcgUMlOgEBGuaKTsqRfFomzgdR7tR5+bKd1ZM6vOmjiLxuAzOkcEOjz5ZBsusBlcduzSMRbdYYq4mdsSkEuu9b1Abc57ssAL9iLm9+PQIzbNwMc2kY39pkNV08vKplhAY4xMAjMANhOgACLQ1Q17NH9+jk/P994DbQ6jnMSBoNRJFK91LijDNc+TVixEwHbi0zCyA8PD0tgGj0OdHpqdKwMSPUcCTMjHS9Xtmu/Ay8GgNPT03VCuQoA2uYZ1pNJNvbcFm7TEhC4rDAIzNSzcyDghA0U8K+T4kjumEXGBjj6xHX8Yw/VgyN9Fp1clGBdQlxWWnXV8rIXU2R/2aZ1Z/0yS38V+EbXaRuqJxOu3t77up808se5cP7Yz4BlJC44VG3MQLBij3pdBSKu/CzwaN0VC1XZKRDIqCc7XxapNJqpYWcyonRaptJVTgLOMAYtk1ejzegdUwL9MChkC1G07KWOr0ar+064z1yfjJyTp2bh9NFX4fjh/LHNxiLLn2Ttzc47ducifraeQeuq7HOGbbkyub1/7phAAAEbsV7DKDiieVnkjogaBhdlqdPHtIFfjuqywyMAyJ5Xq+6c8OGP/ouNrkpTqq/TEGfI2T47QQXceizLp/AUIdrJZbOuJycn68jPDIDBIBhTtRgrE+eo2o6ROCCogkLVj5muIxbG4/5QgoCjhAwCCgB8TfVx5evguAHQiMdgwKCTAUrGVhgA4m1Juladt9F+/ou2eM9COET8p53+jRtHUQUbbg8bUPYERPfdOGh7ta8VoFx/KWCdnZ2t3zJ19+7d9b72G6/Mi08AADD+q/QRE8iu537KFjO5+5ZOO1yfuf0RSwvZGRAYdYAa1qismetCeA7Fb3qZWS/A58KZ9HrV20VJLoujlrs+HJrBkB1F/6eBVxLyOnjtpzDa2HesaaZNDHTuGkfRnZNoOQ7EMpCpyuRgwsfdVsfYlZ2d0ylBxjJmAECdXu+vyqyYD7BDIBDiOiNjBixZQytm4AYvjrMjhuHx/867ujlaO2TW+zgi8SMdR9s1echydnaGu3fvriMlryVgx3bg6AzatS8775w/HNZNPfjaypHcMb6Po7sCT4yf/qO10uOZYKFglfVZdU8FAJmwA2sgnCnLBZpMdgYEKmW5QVVnKGV34JGJUt6Q/f39lEJnHy2Xy1fKGEaierNRc5Sv9k9OTrboqIuIbKT809Y45xa56D63U/tD8xxuWuPGIouirCM7+eHh4Ua/xb5rM7O0LDnqIjkHhYxlZKDJbciYiRMFWbX5ihXw/bNMeObNQu8C8NcBPNF7/yurY/87gP8KwF0AnwLwN3vv/75dvJr84wA+sbr913rv3zulyUAUvflYNSDZ/Rk1jWPAvRWEI+dTSu6mEVmmWOvUNrg6T09P1wkyPcfRMAMcXngUU4fQQftG+4h11D7Vx3ac7+DsPevM96vT6JMPdlSN8pVw3zimktWv41SBgLIrHetRQlLHXp1ebWRpOZXMMIG/D+D/BPAP6NgHAby5937aWvu7AN6Mi/8dAIBP9d5fPq3tQsnofHaNM9ZM3IDHQIbBMDNora0fVUW5GWXXgVSnzCimgsvp6Snu3r278YgyrmFnU3HJMm2zA9is//W7A0IGAAaueGu03suOE/3Mx6MdfE0WvVWn0FUfxTIQcKLSJUndo2C1Nz2/hAnMRu641pW1pIyQS/35SO/9l+nrrwH47xbXvF3Pen+EeNz5DggyUNDvs9MD/hPU3vvaQMOADg4O1s7HeQMunw1X/5uOKa8ajlLXmP/fuXMHd+/eXX84Yx6gxB+l1rrIiOvUMdFEmpuyVH2qjEKnP9xu1ideLBufo6Mj22dcdmv3HuvGExcGaXVWBm03LhrR4z6m6Qw0PNaak9F+1X1n95cFjZlpQ8j9yAn8LQA/S99f3Fr7FwC+AODv9N7/WaLk+n8HnvnMZ05XNqJHjrJWxjpCTgaCkGAEHE30eTXXF+Xo/FYf9Tnj03YECCgABCjcuXNni3Lzo0UGA11rwIbuphU6Xaj6l/uP286Or/3MOu3v72/9D2WAAPefBoEAAV5PwDYROnJbeCqQAaNz5MyWHJirZDbI/ZxJdk7rGZUTciUQaK39CC7eKvzTq0OPA/hLvfc/ba19A4B/2Fp7We/9C3pvp/8dePTRRzs7ywyCuQZXnV0BwAhRuXznCAoCLqOvkYENXv+hVik7tyUMnKP/nTt3Nj7xpIDXHwCbyUKuQwGIgSC27lgGBjomyqK4TD6v/eH+gzLAqloiHVOmAEdnB9oeHZcsd6O2xOOrgM9gsMTWeD9jDVpO5jcPlAm01l6Pi4Thq8OD+8V/EN5Z7f9Wa+1TAL4awIcvW09S9xDh1DCzaKUd66Yabl7IElOCaqGPGsb+/uY/1PK76BkUtM3xFIAXzdy5cwe3b99eb3laEODE0c6xEo1aCgAOODMwZWdw+YYQBcbog1j5qCBwfHy8NR1wRn7nzp2N+T2PO48HBw4FAQZEZRIuAcw25qYDPGVwfcHj7ETHQfv5sgAAXBIEWmvfgotE4H/We3+Sjj8HwOd672ettZfg4s9H/uAydUzosPW9YgLZxwl3uGMCSkEVJCIyxLQhBp+dQp0vO6btjXlspnPoE3kKTsK5uXk25836k/s09t3YaARl3YMNcFuZBQQIxHSAl0PPgpbTI+6N8cnOM9Bw3/M+t5tBjkGAQXfEAFRv7e+qr909S+Syfz7yZgDHAD64qjweBX4jgB9rrZ0COAPwvb33z11GsZkpgV6/0nfo+BUgOADI6tL9qD8ivVLBOA/cWyjE0wng3uMs/jGM1u1+OBNlx6M0Nmp+SabqonNeB3Z8/SgJGKLlqaPolMRNjzjSRx+11jb+es6NAy8pjukQ0373ZMSBs3PGbAEU96cCDJ/PoncWwGLr7Ez3M/AYyWX/fOSdybXvB/D+qZonZNYRgW2EdlOAyoAzIKjYQ4bM4QAKAO4epqj6CJKBgq/n5978yCscC8DasfQlG85wY5t9+NqqX9iweZ7sxlCjruYmFASyd/7rsegT/YERO1/0kUZr58CZPel0wE2zlAWN6Ltrm045WJzzOz8Yyc6sGARqNOTzIxof+2yICgCjacHsIDkd3aCwTnw9fw/n3tvbw+np6UY2neuJcvR+TUS5hUwMCk7UMbQdygZ4qsNAoE82FFhiClA9GWEq7YCM26HO6lYpcvvU+R0b4vZw2awHj7cDEQUTZxdsHwqyXG91b1beQwUCV5nTAD5xAmw+4575ZGXPdKgiMxsIO4MmJ9moOa+QPd+O79r2jHoze4iyXZJKncO1J/ozc0C31bI1DxBAoPW7/s/oeBaxFagyXTLw1n52jEr7PivTjZ0eG9mms62Z8ivZORDIDCiOzYKFc5qZzyxV0/PZd1eWGrg6m0ZXB0IusmQUXuvMqK6Ljq4OBpAscrn72DF0KqBRM4vCo6mdGyNmJcxUMkeadVzXvpHzO/CqgEvrcUncrN9nZedAAPDz5ssyhVnnn2EDrCtH31E7srJHaO70qkBSRR3f5Si0rAoE4lrHNqopBpfLIODqUOPXtmf9yLrpOQcqTqogUH1X8HRAwLrrkunso8CuSU3OOWS2O+M3OwECaqC6raLzjMygLucPlHppGdl8WeusQIXv0wgY93PE5X3e8v36SMsZqtOZ2+umA64PtY3ad9pOZ9BazxLg47JV/4xFOODMwF3ZRTaO2ibXVrU5zVXo+hJdW8JLvSswd2M5IzsBAsC2wWaDtkQyQ9JBqQw47lGjyhyN61Id+JrMWEYJTNeecKhKv8gFaMKORZNmWk+2dfq5vnHOwvsKypnzu3L1O/eBG3MtZ1RHJSNwC+Eo796NqL+ybK1trCSNupgRsP4PNRMA7q3Nj0dBGX2rIutIKsMN41Cqq+Kop9uqoS1hMlp3BgBRfmWwWb2O3uonjqtDsU4a6TJn0nKdDtq2TDjaOdY2EqeDA3W2CwWnaLurl3/AFLagr0XT9y3o0wxlV9HOmcTjUtkJEAjEY6ki81LJKKwOqoJB5hDOsTSia2QetSWL9jPo7nSN+7Oyl5TjHEPZmmNvI3EO7JyRr83uz1iDO1aBt7u397613iIr383t47cM/HFPOZTJ8fTN6e7sc9R2JzsFAs4ZNQplyDeDiBkDiLrco7VRPaoTA4GLlHGP6uT2q/7SrWaMs3rUcbVNmVE5as3nHHOr2uAcmOtxbCwrb6YeFe277N4saGj5fF0s19bfd/DPv3mdR4hOk5jyc3/xeCszWNo/wA6DgC7ECKmAoCpfRRcSxZzZ0WK38ksZQcUaor4KCGJbRVPnrBUL0LpGxjACA0f9tR80mlVR0wGAAyi2idn2zERABU93Lzu/i9p8XXxOTk42ftDltmrHbvGUsxHWubW2tQRaWdFDywTCKStkG80t9frYZizAHYsO16hVsRHVK8pjwGG9ZgFA63Hr0x0IzEZmBTNtkzIcboc6b8YKMn3ctETLBe4lN2fa485lVFqdR/XlZJ4rW6+LnzA/+eSTePLJJ3H79m089dRTuH379vpXntFHMY78q9Ks/OgDDU7uqc8s8AM7AgIA1iBwfn5v/TwbZBZRsmiSGZwDAXdcj1WApB99hJMZmOqWSeUkGQDMimMSmZNkwJu1repr1T8rN5suXLaNWftUf436o9WJnAR86qmn1p9wfP41J4uWxYDD+vFy8qw9M+sGnOwECDATiFd08fJZh2oVOCgl4nPu2ExEVjDg75mxj2Sp8/NxlwNYUk9Vtn5nI9PVjFpP9VFdRiBbMS89rrpUbCYDzyjH/eYia4Nee3Z2tgECTz311PolL/z7BdZfbVmXkXPZFUBnuY2R7AwIHB4ebjw/zd5yE5JRSXXmbD9zdN4fGWkGDJnTVU7vjJhBhuvPFv24cpwevJ85RBU59XzWb855eNxiyjUSdw0DkmMYro3ZbyJYd9YzYwJ8rT7jj3wAf/ifkrXvqro5TwVgiyVwG/f29rbY86zsDAgcHBysG3h6emoHTI0IyB8JOYeswCCLWhlYOArJOs4OROb83EY9P2vES1iJo/pZtOEE6ggAMufJ9HRty0CVdcmuUQAIhpnpki3hdbYE3HvPQWT94ylAJADjnY+sC/86tOoPTVSr84edxe8veLmxtr+SIX9orb2rtfZEa+2jdOxHW2t/3Fr7yOrzGjr35tbaJ1trn2itffOo/LUie9tv1VmCaiOnngGArCw+xtfMXltJFYVHkXhp+1nvkT5Ol4pWu/o0kmoCVvtypq8cKLFzV3q680v6z+nOIBAJwKeeemrjnY/6HxFZe0bjGfXpqkN9fwK/SGXGFi/7vwMA8BO99x+XQfpaAK8D8DIAzwPwj1trX917334R/uZ99hGdG7RRg7IoPXICRV6mbhxtnPGqIyhLUWfW9QhRtmM3rs3uugrknIzA1ek8++zeAcGo/hlw0rHlManaWTmb6jvr/HxvOGH2Kvje+9Yjv9DDLaGeER7rqD9slV9Syy92zeRS/ztQyGsBvLdfvHD0D1trnwTwSgC/Oroxc3pOooyYwSjSOyBYGgH48aU6bubE2k7dz/Ry+6zzkjZHfVn/Ob0yWqkJwqwvOWppWdoO/s7AV+mlfVJ9n4m4+j2bEmgb+A3QEfmZATg9oh/1V5uOWen9rF88NXB9GuVmfRZylZzA97fWvhsXbxL+wd775wE8Hxd/RhLy2OrYljT634FnP/vZGz9wmaGfIW7wK4o6AgCd78Z3ZQMZAGR6rNrM7U+vc45QAYVrgx4fOb+LshkgZzpwnZpdZ4NUyaKs9kEFqHpPBoAOqOO7PoniY1mZnMdiJsD/uKT1aV/EVNgBqurPthEfTjwyOHMAdf27rj89U8tPAvgqAC/HxX8NvC10Ndfa2nvv7+i9v6L3/opHHnmkdHgHAJnh8LnMSeIYX8/XZb8pqEAlQ+4QbotGAPdRepg5sYtcuu/6wPVDpreLVKPfzDsHquapo/FSfRSc9DVlM/mlzBaq8Xf9HwuEIi/ALznl9xy6P2TNxn5Geu8bOQGul/MPlV0Cl2QCvffPxn5r7acA/OLq62MAXkiXvgDAZ2bK1EHV6LOqNzVeF235Hh1oZ7AacV2Um5URjdXIO0LsGf1HBuvKcbqw3hkoZ23WNjETUFB3x1mXTC/WIetTxzwyJ3agmV2jdfA/TqleAU5xD0dwboc6P9s+P8nQaZWuQo164v2U/J+ZlVz2fwee23t/fPX12wDEk4MPAPiZ1trbcZEYfCmA35gscwsIsnXdzpC5nOz62cg+ipjOUCon0vMzAFAB0ky0rQbeOb/WnUVcTd5W/R5b5+xaFzvLSP+sTm2Ta9vsWLt+dgvXqleas3Bg47qcvTvg5T9ndX0Y5/TlJPo3bE4u+78D39RaezkuqP6nAXzPqmM+1lp7H4Dfw8Xfk31fHzwZ4IZUn1X5JRuYocxLHD4zjixiz0TIDACcztrGWQDgcrWurF9Uh5nxcO1246V5FjZkjmhcxiwQ8PV6b9yfLSxy/TiaBur6BwYB1x6nTwYA8Yn8gNofv5aM/wtTAVrfUzCS+/q/A6vr3wLgLcOaRZaCQAYALrpph/KA6uO60eNAaatth2uTOz8qi8+NjFL7xUUKrcf15cjZNcpmABDl63RAHZb7X89lkrGO2HdtYGd0QJAFA+1rfU6v/2ugqzk1YLgAEv2gL17tvW85s/YP1xH7zAJmcgw7s2IwOoIHTxuQRWwti8U5djg/l6nlZVHMAZDWN4qWem1WDvdNpp/2ixqc3sffR33J9Y+ALGsffzLHzMCrkorlqIPwfgaCM+DKzs+UmwEgVr5qkpb7uQp2vGhO72cQc0EqACDyAbyUuJKdAAFg8x14DhAA//zaOQuwTd1b23xXfiYjw4h6+NFhSOawWYRz7EXPZe12/ZI5u2ubO6/1V8YzUwbXOSpndCx0yo5VjMTpo0CoUwJ97VfGuMJpDw8PAWBj+a46MZeh0V5Bin8mHHVwYlD7KWNgDx0IxNY9Gssaohlgh/ZxXNeAj4wzYwPV2gEGHy5rKb1VgOBt3DNDdZ3RaPmzDq3ljK6pmNOoXgeaeqxiXAqMLnKOIr1bLOQec+7v7+Po6GiDCXB52WPb0PH8/HzjH6eYFbjHnG68uV1c30MDAtl0IEN5vZfFGZbSqJE42qWAUAGA1qODNgNsuiKP782OOfCrHHCmL7L+dWVm9WgEq/TLgMuNe2YjmR5OV93XOX8FANEu/qt0dfasPC6L/1dSQV6ZAJ9zwcUB2kMBAsA9w3fzpJCKImcOwPvVs2AWvU/ZgAKBggHrkAFAXJMZcTbl0Xti3sf3zTKd+yXOkd0x1pGlou5xPgODOOb6K4v+bt8l/RwT0HGOMVC9s/LcW4bjHL9VKNrDv9nI+kjtgusfAQCwQyAQogPqwCCTWbSPTuI8hHNide7M6R0rcHVrtFaAWBKh435gc0qUPQrj62f60omWOXJ2x4j043I/er22d1ZH1SP7OOpf5QNUtyogZX0X4xTl8z4/ctR+07LYppQp8F/WV7JzIADkjwvjHEuG7LzlztNB584KOsfPsSvD0WkBH4syHLtR9M6AIHOwjCUwU1FdAWzlWmbANetT7s/s48aU6a0u8XUsaUbPDGiXOP0sC+B6GHydY0dZvW/+9Zj2H9uN2qaOb2bv0T/O3kayMyCQGc2oQc5Iq0iqhuCkcv4s8uux+K6GkgEB61Y5lJsSANgAHlemHs/6WvuJ+zY7N6OzAwBdIMPlc9tUV3edAmmlt44b7zuaztexDs4GHKBUQFTdnzFHbROP7UzgVNkZEFDJQEDRULdsBBlr4AGq6h0BQJUkDFFkDj0cALg2OIN2FDn04LZyRItreq//kVdlBlArANB+VcdnMNA28H0z4vpq5PTZFKCaCjBTdI7s/k2oAqaMQejCo9hWZS1hACE7BwJhKGwwWaTSbWaAfB07btQXW6byWaRTp3agAGw/umQ9HAAoGLg2ch/xln8yqu3ROrMytK0u/5E5xQwLGDEBfkTMQOAi2szUQHWbdf4simdjp3W4j7NL1186HRm11bU7ypllAcAOgYAa78zPK3Wg4/4MCNiwgwUwjc6YAw8y/3JLgSBjA9pOdUrVuYq+XI6rk3WK1WNcj2ujc353nn+Z5iJcBgAa/d3PZxl49RGpAzcXEFRnpvLqlFkeQAHDAR7XXzm9u9f1q+qsa/6VATsg4mNsEw/VI0Ie5GyRRIjrTFcWC3cyX6fOw9c6kGFHUcTlx5wBGE7vDAj4GmdIem3sK4iE8/PyUTYa107tm9CfX4rpHEZ11vGZnQbomCkwO+cfjfOs87s8gHNmHcfopwwAVC/X59wW7ePe703dMmbJfaDXzLKJnQGBEGc0I+o6EnXgkNHzV438/ChRIxWDAzs47zt6PssCHNCxU+j8kYEgjJP7UiNJtC+uZf2j3bGdZQKhSxb5Y+ls9SMXbqcDAO4/HedssY8Cwsj5R9E8yyG4MRzZbZTFKwh5JaFrvwN3rm8kOwMCDvFH04EQFyH1nDppdK4movReBwZRF5epaw0cEESZ2byyorhZn2V0WaOtsh1to05rgHvAEODAK+J4WzEWXosxYgKhg7ZP9/l8BgCOtSjrUSZQObDWkTEo1g3Alt1U48o6Obbnys+u4TEayU6AADtRtlZaPw6ZVSpKBmD4k0uXIGQHYSfPwCDudY7P52b7J75XHwBrR9M/eq1YleszpqOcR1EnyCJhAFKVEHTRjPf1WJSdTUmqBF98+GfA1Wu/XOTn9nF7K1BU9qH2xvakQMDTgcwedX/G+UNmXiryLgB/HcATvfe/sjr2swD+8uqSZwP49733l7eLtxJ/HMAnVud+rff+vTOKOCbgWIEDAic6eCr8U8t4+wozhNAlQ3++XgfbRa+Rsyu9zaIe90fWZ+Fw8WMWYNtxHRhxO1lnR8ezyMqOpKCki4Tck58s4nE/qo7O0ap1/8oA3DsBtL2OpVW6OJ24n+7evbueDrXWNt4AxLqEPlkdrs9Yxxm51P8O9N7/+9hvrb0NwP9H13+q9/7yaQ3ulbPeqqHfD1YQwnT+/Px8o/PV+ONaR/VnPozuVXvdOXWk3rffyOOcP66Nqc7BwcG63WqQWreLalkU1v5Sw+S2KQtwLwEN0UeEro8yAFIn13MuIchsgINN9pia2+36IbZap+4fHBxsRHfXl1G/Mh4uy9mO7o+Y5pX+d6BdlP4dAP6LUTkjGTEBNnSOUAPdt/YzBNUIqZnxpY6vTGGka9Yf/N31kesz1b+1tjXvdfWoo6iRVSCrAMrP/av1AXxtSAUEPLd2zl1Ff2YIygQi58F9595c7No66kenI+cKMtbCuug9yrq0r7LH3k6umhP4TwF8tvf++3Tsxa21fwHgCwD+Tu/9n40KqQyanT97nDRTPuARXCMG37O3t7eOpHFs1vnjO281gx86ZGwgO7YEBPS8vqaK2xX6cT84IK3EjVnGArQNI4COfefcFQg45+fXcnOdoePh4eEWELgoq+OiOmaMQNnt/v7+1t+Iqa07AMjGxPVjJlcFge8E8B76/jiAv9R7/9PW2jcA+IettZf13r+gNzb685FHH31Uz5VgwPPuSrij1YhCeu8blFANIso5ODiwEd4BAbeDy2IgUFDQe3QQs4ijbdP2czkKFCocpZQNZFSf6+dzGQho5HdUOmuPc65sWqDO5+h/AGLoE3mUg4ODNQhkU1IdCx0XxyL5fNRxdHSEW7durf+xKP6+LP60hBOUDiQdY+QxmJFLg0Br7QDAfwvgG6hxdwDcWe3/VmvtUwC+Ghf/UrQhvfd3AHgHALzoRS/agjM1pCybnEUPHjide6mRR0TQqUYYOAOAi1ya2WUGwNcyRXNgpg4V30fbuNYBixovTxO4HsCvfpsFAR2HDAgyMHNgwFu+roqyDgzC4fkPQZRy7+3t4fDwcOOjOYsMDEbC4xX3MAC4fzS+e/fuBmtzQKvbWX1UrsIE/ksA/6r3/hgp9hwAn+u9n7XWXoKL/x34gyWFqlOoQcXW/fhH52hsgMBm4oudNv69RTtU6Tt/NOkXzhLntE0ZQ9B6XD8sAQAFn9hX4FOjjDIcrXZTngwEKiBQpxo5vpuKOAComFmAQDh/AEG0nwGKo3OAgBtHF1x0rDK7jWOakzg5OcGTTz6Jp556CkdHR+t/MtIpAtuGS5hn/VfJpf53oPf+Tlz8+/B75PJvBPBjrbVTAGcAvrf3/rkpTTbrtJ8sL8CGp4PAIMCJlRiI8/Pz9R9IRhkaubhujfxcf+99a813DJzLBzAjGOULKgCIYwoA6nCaaa5AIIu4mdHxdW4c1RF47DLKnG0rx2fn11wA/1UXgHWGPqZ7wQBi342xApq2QduaJUVV39PTU9y6dQtf/OIX1zrEX5zr/D/rUy6XGdxILvu/A+i9/w1z7P0A3j+s1YiL4Nk0wBmiRj1HQ7lDopOU9sZ1wQxchNOEW9Sp0VgZQYhey1t1cOnfDafn49V1bBx6HbdZ25Rt9bqRoYVRKlNhB6vofwUAylw4067Zfy4rbEOdX8G/0slR9KovHEACWNd369at9ZgxOwmwib85V12iPWwXs2MD7MiKQeBeIypnZzDQfXfd6IcqOkfn33CfnZ1toXDczx0fx9jIo8zYqtFye4FNqusiZYgyARdR4zr+aCKURYGg+mSiTECNkfsto65uP4vyGQCo87vFQOFgHP2D/gcIKPhmbXXneKvRmLcKtqHP8fHxej8+0Xdho8xuFJi5fLWRTHYGBJYwgRmgYOdndFfwAO4xh4gYQc+UITAFVD0cE3BsgKcU8Z2nAy66hLjBzWizY0oqDhA0OrtorfeqQ7oxdWPnJAOCaoriQECfAnAeJGwiGADnADKa7/opczIe/3Bevi+z6dAhdGIAiH4NNsALnLi/KyaXyU6AAFNWN5/KwMCVkc3D3DNfrevu3bvrxzQxf4wBVQOJMnUeHufc0wF2EKXo7OBclovAbprgDHbGIDIQ4XLdnNSBgIuQCu48xll7VJ9RnmKGBXBdHGkZCEbixkL7m48zEGQffq8l5yLOzzdXe/KLRiKPpTkaDiYKCJXsBAgA28aSRXwHCsA9p2SkVwBgiqWMIK7t/d6fTEZe4OTkxAIIGzMjfrQjmwJohHNzdjc1iHpUnBM6J8uouIt2mfEqAHD9YaxaL5en0wLWS8vUfhoBQERIBQQF8tbaxqPAiLquXtfvDjAd/R4BgbadbTTu5ScbnMgG8pfYsk4PFQiEVPSenfr8/HwjEmdTgOhUt2Q1Bj72M9YQ152enuL27dtbkYbBJUBCKaH7MACw884MHjuyRmzty+x7iItmzqj4Wt6Ggerc25U3+mjb2MHV8eO7vtOPQZfHmD+6HkCd1/X3DCvTc248FXx1+sjrA0LXW7dubT1S5AVFGTse6QjsIAiEZJE/HD/oEjtTNhXIphPhsDE/ZLCIDw9YdDxHmNPT07UxnZ+fr+mcOn+IHmMj4HY7ys1bFqXsmRNy31Z9njEGrSu+87NsfgbvHNuB/AwQuMjPjh/sLebJMa4xr1abUOBmNlb1TybOASsAVpao5TA7YBAIAIipqzJG148PHQhkTIAHUQ0hOsFNB9yPQBgM2MB5vXiAQjxX5hwBU7TDw8ON6MdAFYYFYMPAGAQ4Crj2u8ir+8oGuBzt2+r7KO8SopGZI1MGAlqnjq0DBdfGDAAYmJkVMpvTx25aP+unU5eRVCCQsS83ZgxiMSbxZ6cxFYjVhfwoM+vrhxIEQjIWoCCgTEBBgJHeGRx/+JVccd/t27cBYGPF2dnZGQ4ODrZWckWEUYDKgEAlm+M5x9fI7wyKt1kfs5HwenfOt/D1GpV5OW4AZtxfGSBHa+eQXLdro/s1YHy4TF78ozkhFp6jLwUAbpO2T/e1zhjzbBq3v3/xZ6cxNgEAEbAyhjcCfJadBIEwAjbGKkkI3EsMZjkENTC+jq/n+qNsjv76OC+cQBclMRDEf8wp5dS2RN0ZCGg0dGBwGRCIfddfqpNGXmZFAQR6Dws7WwCv1sl1O9ajQUBtRp//R+LPPXvXcrnflwBA1cdOqmmdTg9dMjM++tYkV/9DxwSckys9dPc4FlBNAxxdi/s5Owvci/BHR0e4c+eOdczT09P14qKYJoTxRfJQAYf1ZmNmfZyTOyBQUNCyuJ3uO/d3xZzU6TUKR/0OhDTaaZ4nYwJ8rwIfgA0AB2CdRRkesD2tYeDJwCvryxkZ3cNTRG4/2yEnu4+OjtZTVE6ILpWdAwFgc529ztkA/7gmA4DMoDMgYDagicOjo6ONNQQ8Bw6WcHp6irt3764H6OjoaA0KYYABMnt7m3+I6pw0eyyWJcpcJIntyAiziMyApfQ/2q+OmQkzoajLAUDGZHRq5Fido//KcrR/Af9vUZeVjP7PAAGzkiiL7ZvXNwBYPyXQ/p9tw86BAKNebKv5pdJZTvzwwDtm4YCADZGdNn72eXJygtu3b+P27dt46qmnNjLTMQB7excLj46Pj3F6eoqjoyOcnZ2tDTMGOfRkx2DJ6L/SYV1KysBYbaMO7kuXg2Ew1ew0//qyEpcXOT8/tywgA2gd85AAaef4DADcPo6aOkXjPE6cfxCRP+sbZgKOpfJ0h8twv6ydmRbuDAg4BFMW4BJkfI9STJcL0OvU8JhWAvd+981JsHj2HNQYwMZcGcD6Hue8vfeNPAFPD7gvHAPIHpMxELCBqzO5qRWDl+tD/sRvKuI38AECOpePPuGx0gjOydMKBLJ8CY/Z4eEhjo+PcXx8nE692La4fDcVUCD4UomOiYIAM514LM2PRjOdq7bsDAgA25nvcDp+Ps9GrpHbzfdDXGdmjECvj6gV9/Bbafb393Hnzp2NxFjvfSNJGMkydmZ+EsFsRfVQqq1t135TcdczEKqzuT7VvAXT0qwPmWJnYxz9Gjo5UNfx0qkKr/qLRGCWEFbJxn1mOjACiMr+uB4+xmXydIABSacF8bp8BbHMHpzsDAiw0fDqM444+mpoZyTV4DnDdsDBW3b+OH58fAxgcyFHgBW/ESYGI747AMgYS8aAuC3qdLq4Se+LCBzRs3o8VxlxsKWg8m4slQVkY66g4KK8OjX3G0fFmA5om5idRL2uXVlAcPeOQEKvcbaVla1shPuSbZ77RgNHyAwQzLxU5IW4eN34fwDgHMA7eu//R2vtUQA/C+BFAD4N4Dt6759f3fNmAG/AxYtF/qfe+z8aaoJ7xsNJp3j3GmejNbmUGa5py0YHZqisrIHr41wFAwC/IirWEuiPWIJOB3qrgTtjz9oQomxAM99xjoVppT4244/LSUSfs2OxM8f1+uhKdXDMxj0a43m9m+szEHA+SBmNqz/r2wwEluYGnMNn+06v6Bu2QR4Dnvbo/QwYI5lhAqcAfrD3/tuttS8D8FuttQ8C+BsAPtR7f2tr7U0A3gTgh1prX4uLtw69DMDzAPzj1tpX9963sxakOBtFOHw4ljIBYNMZqmlAiEP6igqyI7COUR/P+ePNRLGIg+fM/NgsnEOTbwwCbNw6kFnU4tyCm39rookjrHudFrMLznvEvJPZBNcRjh9PDDQBx+MdejPYOLBi59eXf1S/C8nG0kVyPp/ZkQOAjOFUUX/EMLgudX4+rixS7TXT2cnMm4Uex8VbhNF7/7PW2scBPB/Aa3Hx2jEAeDeAXwHwQ6vj7+0XLx39w9baJwG8EsCvVvWE4TCtDmfSnIACgNJnN1gZ8o9EjSPqZEdhCqoLbdxjO52v6bxf6aomNNVYoz5mAhXN1rJZb+4XNjwWvqe1zdeYu8dsvK+6a8TOor9Sfn30N5PncWOr/aFjXTn/TNl6DZer5fG+m9MriI1yHlp2JotyAu3iT0i+DsCvA/jKFUCg9/54a+0rVpc9H8Cv0W2PrY6lcn5+vn7cFlFVGYA+huP5rTO2uIajjQLIiDJVSM/GEdSaB4iNOVhDnOO6+Rgn4BxNVkfhaBvnuB/iw7Qy+jsiPNenNJrBSfuFj7s1/O5pBbMHbotG/GAo/Lxf5/sKVBnQ83hV57kfeUwqO6jEMTinE4ubvrEtcJ87e2AGmU0FnUyDQGvtWbh4f+AP9N6/UKCgO7GlSaP/HXj2s5+NL37xi2smoACgj9cculaOrFGRj2cJxRGV0giii4wYAGJVlxqszou5bmYPGjU52vPWJTvDuNWw+K008V2jKuuiYMqiv3mPaQFPEVw7dDqUrfNnVpBFvsrJXV/x2DkG4MY+c9rsej4+Cyp6LmxAgU77kT+cSJyRKRBorR3iAgB+uvf+c6vDn22tPXfFAp4L4InV8ccAvJBufwGAz5jGrv934HnPe15/8sknN6YD+jSAjU87P6Odcczdzx+XQS76ws5d4xwn3GJttwMy/Q08txPARu6D2+aSXOr4bNB7e3tbeQkFIAYAjijcVgURvl9/SqzjpPpwYpLr07f9up92u/wPA1QmTn/tv2BDHCz03qr8ERDw9wpQ3DkHBAz6bho6q/vM04EG4J0APt57fzud+gCA1wN462r7C3T8Z1prb8dFYvClAH6jquP8/By3b9/eMCjOsFcRmYXpPx/jbfboKpsDzrCOOBYD0XtfRy52dHV6fnIQtN0xBgdsvDrM5Qn4XFzv6uFkpXM47QNmKdwmHjNlMKMkKE8BGATckxONrAoAIzDQcXPflUEtKVPLy1jmaN+dq6YDvB/X308m8CoA3wXgd1trH1kd+2FcOP/7WmtvAPBHAL59VfnHWmvvA/B7uHiy8H29eDIQCt+9exe9940EIK8J4IZzI6NzOJPK2XCuI0SBhTuWpTKSGerIA6XlOCdR53KZda6LqaKrn+9xmXqdJikrcDkXBZD4RGI32q0U1TENlwDMMv7VlE2BMnM6BXfHqLRMV/6szF7rgCBjYDyGzo702IzMPB345/DzfAB4dXLPWwC8ZUqDi+vXK+3YyICc7sd9SqOBzTf46uCpQWQUkK/hfT2WsQZ2MI3srGcwCG47Mwe+R3WP9oeh6rTGGbkCAjOk6ItIGLqPtkl1jHyFUnmlrA4IHOXnvsycSh2VZQQQ2p9aFrf7MpLdp+3RcXZg4PrBAYGz5Up2YsVgMIDYdwnAzBizxjIzcECgqO+mHSMQcOccELjBdYitFFvn8QyOnDHW/shYShhJtNuV6+7Rstx0RaO/e4nHiBm4x5UsGeXPGEE1Xq4s107XJ3putnx3/wwYZN8r/1giOwcC8R3IG8nXzUSIjG6x82l9Wo4776J6Fo1cmeoYGRtw+1Euz/O5XC7fRQvuB5eL4D7KQCCEf6MfCb/qzz0zIMjYR9WHrI+L+OrIjg044OC2zzCBzHYqJuC2fN59XL3VZ0Z2AgSAMbK6RvE9Olfi+xzSjwZnid4MNkueMqhDAPfyFeHYPF+Pff4tgpbp9l2d3FZnYG4qEaKJSJ3P66u8+Y2+rINGf9VTxTE4PZ+dq8S1X+1maZlR7sz5jPKPAED1zfxjdO/OgMAoCo+EO0uXFnMHuWTXknqyay9TTujN+qve+vgxHIanCzMRIiK+OjA/GVAWoO1yH34k6pJ88V37x7EV1y9xj4vgVT/OXJt9135URjTr3Esko/0jplux4xjzkewECLCxA3UUyo5zgyt65BbUVHVUOo/0Y6kG0z1X13ZwjsNR+FEdwPaPjXga4gBAdeB+4yiuP+bRCO8YmJuqsG7qyI7qZ/3M+8ocKtqesQFlAWp3lQ5LxU3LuMxZ/WcZBLAjIAAsd8JM3DNqNqowYDXOkQ6ZY7jz7lg2KPG9ovYVfVcgyFYmqiFnVNE5An/Y2bMsP7MXde6sbNWhin5xTVV2JhkQzNxfTQmu4vhVeaN+cPctmUYAOwQCbOjR2SP0U3FOF6CgtNolorK6tEOVSTjndM6VDU7GfHRxTEZH3WrA0XoDV5/bdyCQvb5LwdVRU+1HnbJk/VcZfUXTZ1gAX8v1ZUzCXZeVMyOqf7Z19y0NYE52BgSUMl6GGShoMAjE3DccWNcROIPXcllXBpbZwXOOMOqLiq0o+KjDMTiMxPU3t5OnAW5loSb4XPu0DgcC2l/uPufYmcOO7tO6sr6pgGC2nEyWMJnQh+1Blw8/lOsE1OkygxyJc0J2AnXwMNiIXhzds0d+DjQcE2A9+DsDQSVZvVXbWY9sbpndm+mgjzHdh6/L9HIOnQGtXh/fK2esgkcGFiPR8nW7lAEog5i5ZqQvO3+8boyXgs/IToAAUBv9CCAAn12N/eyXbNGBfM4ZuCLvKCpX+ikAZBR2RkZ9cRUACMmcvvpUBquOWjGuOKYOPHMuO1Zdq/qwzu77UvYy+n4ZFqF2e3BwsMF6Z2QnQWBEg7OBzWgxRxJHpZgJKL1Vwx+h6wgElJ5nxuGMIotK7tgIlFwdeq32CT/L53tmouQIAFSnzJm0TnfdjJNX41QFpAoAqvquKo6BxndmAvzbD8dknTwUIOCM/jISDsivyYo5lKJ9xgpmKJajfQ6g3PXKYtx5Z6RVH42AqbrGLegZ0fuszJEDuvawrpkzZucd8GTsQOsa6TczrbhKPVqHu1YZbfbbi5HsDAiMELaKIBqNnMNxp8QUIRbd7O/vbyxbdnVER890bBb1MqdzIFBF6My5LgOSmaM4maGrI1o7mv5UDHAUXZdE32rKUl2jzlkxkiX1Z1MOZw98D9erTFavyWTnQIC/uw7VORkfc3TURS3HCLSOmeSdk8ohXSR0+9VUgM+7fpihu6zPbGTUacySds0cyyj4DCtwZWbOmumRnRsBwYxUrGGGDeh1HNRi3zGBmTqAHQKBEDcI/Iw/uzYzosrJAgT0n3RnltFmsuS8a2tGpfm7Zv4d8Dia7IxxBLRON8dwMl0rydifGnilpyvzstOArA0jZhrf3fsa4lxmh1q+m2qMgkY1HZgZi50BgQwltTOqgazmQew0MVj682HuzCWP2Eb1z9J1BYGqT3gNANebMYHMOVTHikFoGTOAkpXF91RANuvM6nAVCDo9ZnRlvZzorzPdVMGNb6Y766A/kNOxiED554IJjGQG0TP6OEJjfaFJVvbIMC4DBJell1m9zlBmmIBzIrddSrOXXKP67JK48edjIxC6TH2Vzei1vJ2VudUE1yyX7Ty3fx2SgYLTUY/xh2VJnzyI9l9mTJZIZfy7LvdT7yVM9DKsDADagx7MKSVa+3cAvgjgT65blyvIl+Ph1h94+NvwsOsPPNg2/Ie99+fowZ0AAQBorX249/6K69bjsvKw6w88/G142PUHrqcND8V04EZu5EYenNyAwI3cyNNcdgkE3nHdClxRHnb9gYe/DQ+7/sA1tGFncgI3ciM3cj2yS0zgRm7kRq5Brh0EWmvf0lr7RGvtk621N123PrPSWvt0a+13W2sfaa19eHXs0dbaB1trv7/a/oXr1jOktfau1toTrbWP0rFU39bam1dj8onW2jdfj9abkrThR1trf7wah4+01l5D53aqDa21F7bW/klr7eOttY+11v726vj1jgMvjf1SfwDsA/gUgJcAOALwLwF87XXqtED3TwP4cjn29wC8abX/JgB/97r1JN2+EcDXA/joSF8AX7sai2MAL16N0f6OtuFHAfzP5tqdawOA5wL4+tX+lwH41ys9r3UcrpsJvBLAJ3vvf9B7vwvgvQBee806XUVeC+Ddq/13A/hvrk+VTem9/1MAn5PDmb6vBfDe3vud3vsfAvgkLsbqWiVpQyY714be++O9999e7f8ZgI8DeD6ueRyuGwSeD+Df0vfHVsceBukAfrm19luttTeujn1l7/1x4GLAAXzFtWk3J5m+D9u4fH9r7XdW04Wg0jvdhtbaiwB8HYBfxzWPw3WDgFvg/LA8rnhV7/3rAXwrgO9rrX3jdSt0H+VhGpefBPBVAF4O4HEAb1sd39k2tNaeBeD9AH6g9/6F6lJz7L634bpB4DEAL6TvLwDwmWvSZZH03j+z2j4B4OdxQdM+21p7LgCstk9cn4ZTkun70IxL7/2zvfez3vs5gJ/CPbq8k21orR3iAgB+uvf+c6vD1zoO1w0Cvwngpa21F7fWjgC8DsAHrlmnobTWHmmtfVnsA/hrAD6KC91fv7rs9QB+4Xo0nJZM3w8AeF1r7bi19mIALwXwG9eg31DCeVbybbgYB2AH29Auftr3TgAf772/nU5d7zjsQMb3NbjIkn4KwI9ctz6TOr8EF1nbfwngY6E3gL8I4EMAfn+1ffS6dSWd34MLunyCiwjzhkpfAD+yGpNPAPjW69a/aMP/A+B3AfzOymmeu6ttAPCf4ILO/w6Aj6w+r7nucbhZMXgjN/I0l+ueDtzIjdzINcsNCNzIjTzN5QYEbuRGnuZyAwI3ciNPc7kBgRu5kae53IDAjdzI01xuQOBGbuRpLjcgcCM38jSX/x/fWvWXNaN5PAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize dataset:\n",
    "\n",
    "for category in classes:\n",
    "    category_path = os.path.join(Train_DIR, category)\n",
    "    for img in os.listdir(category_path):\n",
    "        print(img)\n",
    "        img_array = cv2.resize(cv2.imread(os.path.join(category_path, img)), (224, 224))\n",
    "        plt.imshow(img_array)\n",
    "        plt.show()\n",
    "        break\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f25d849",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training and testing set generators:\n",
    "\n",
    "# one-hot:\n",
    "def label_img(category):\n",
    "    arr = np.zeros((classes_num,), dtype = int)\n",
    "    arr[classes.index(category), ] = 1\n",
    "    return arr\n",
    "\n",
    "img_size = 128\n",
    "train = []\n",
    "test = []\n",
    "def create_train_set(img_size):\n",
    "    for category in classes:\n",
    "        category_path = os.path.join(Train_DIR, category)\n",
    "        label = label_img(category)\n",
    "        for img in os.listdir(category_path):\n",
    "            img_array = cv2.resize(cv2.imread(os.path.join(category_path, img)), (img_size, img_size))\n",
    "            train.append([np.array(img_array), np.array(label)])\n",
    "            \n",
    "def create_test_set(img_size):\n",
    "    for category in classes:\n",
    "        category_path = os.path.join(Test_DIR, category)\n",
    "        label = label_img(category)\n",
    "        for img in os.listdir(category_path):\n",
    "            img_array = cv2.resize(cv2.imread(os.path.join(category_path, img)), (img_size, img_size))\n",
    "            test.append([np.array(img_array), np.array(label)])\n",
    "            \n",
    "create_train_set(img_size)\n",
    "create_test_set(img_size)\n",
    "\n",
    "train = np.array(train)\n",
    "test = np.array(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7537dcf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(os.path.join(Project_DIR, \"combined_train_set.npy\"), train)\n",
    "np.save(os.path.join(Project_DIR, \"test_set.npy\"), test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9c18fe36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATING VALIDATION SET FROM A SUBSET OF TRAINING SET:\n",
    "\n",
    "valTOtrain_split_ratio = 0.2  # (val / total)\n",
    "num = int(((1 - valTOtrain_split_ratio)*train.shape[0]))\n",
    "train_set = train[:num]\n",
    "val_set = train[num:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bd3093dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22967, 2)\n",
      "(5742, 2)\n"
     ]
    }
   ],
   "source": [
    "print(train_set.shape)\n",
    "print(val_set.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7f0251c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22967, 128, 128, 3)\n",
      "(22967, 7)\n",
      "(5742, 128, 128, 3)\n",
      "(5742, 7)\n",
      "(7178, 128, 128, 3)\n",
      "(7178, 7)\n"
     ]
    }
   ],
   "source": [
    "X_train = np.array([i[0] for i in train_set]).reshape(-1, img_size, img_size, 3)\n",
    "y_train = np.array([i[1] for i in train_set])\n",
    "\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "\n",
    "\n",
    "X_val = np.array([i[0] for i in val_set]).reshape(-1, img_size, img_size, 3)\n",
    "y_val = np.array([i[1] for i in val_set])\n",
    "\n",
    "print(X_val.shape)\n",
    "print(y_val.shape)\n",
    "\n",
    "X_test = np.array([i[0] for i in test]).reshape(-1, img_size, img_size, 3)\n",
    "y_test = np.array([i[1] for i in test])\n",
    "\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4b742cb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(os.path.join(Project_DIR, \"X_train.npy\"), X_train)\n",
    "np.save(os.path.join(Project_DIR, \"y_train.npy\"), y_train)\n",
    "\n",
    "np.save(os.path.join(Project_DIR, \"X_val.npy\"), X_val)\n",
    "np.save(os.path.join(Project_DIR, \"y_val.npy\"), y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7f87f0de",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(os.path.join(Project_DIR, \"X_test.npy\"), X_test)\n",
    "np.save(os.path.join(Project_DIR, \"y_test.npy\"), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0f19cc85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN THIS:\n",
    "\n",
    "X_train = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"X_train.npy\"))\n",
    "X_val = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"X_val.npy\"))\n",
    "X_test = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"X_test.npy\"))\n",
    "\n",
    "y_train = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"y_train.npy\"))\n",
    "y_val = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"y_val.npy\"))\n",
    "y_test = np.load(os.path.join(Project_DIR, \"Numpy_Arrays\", \"y_test.npy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c00ea857",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22967, 128, 128, 3)\n",
      "(22967, 7)\n",
      "(5742, 128, 128, 3)\n",
      "(5742, 7)\n",
      "(7178, 128, 128, 3)\n",
      "(7178, 7)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "\n",
    "print(X_val.shape)\n",
    "print(y_val.shape)\n",
    "\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "\n",
    "# We skip normalization of datasets as my PC is unable to normalize such large numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3c25ff74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR! Session/line number was not unique in database. History logging moved to new session 313\n"
     ]
    }
   ],
   "source": [
    "# We shall train some models with transfer learning first:\n",
    "\n",
    "img_size = 128\n",
    "from tensorflow.keras.applications.resnet50 import ResNet50\n",
    "import tensorflow.keras.applications\n",
    "\n",
    "# resnet_model = ResNet50(input_shape=(img_size, img_size, 3), include_top=False, weights='imagenet')\n",
    "mobilenet_model = tf.keras.applications.MobileNetV2(input_shape=(img_size, img_size, 3), include_top=False, weights='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f1a7363",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import np_utils\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.datasets import cifar10, cifar100\n",
    "from tensorflow.keras.layers import Conv2D, MaxPool2D, Input, UpSampling2D, Flatten, BatchNormalization, Dense, Dropout, GlobalAveragePooling2D\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0b7fbef4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenetv2_1.00_128 (Model) (None, 4, 4, 1280)        2257984   \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 20480)             0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 20480)             81920     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 7)                 143367    \n",
      "=================================================================\n",
      "Total params: 2,483,271\n",
      "Trainable params: 1,224,327\n",
      "Non-trainable params: 1,258,944\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model1 using the mobile_net as the base model\n",
    "\n",
    "for layer in mobilenet_model.layers:\n",
    "   layer.trainable = False\n",
    "for layer in mobilenet_model.layers[-14:]:\n",
    "   layer.trainable = True\n",
    "\n",
    "# **IMPORTANT**\n",
    "# let's visualize layer names and layer indices to see how many layers we should freeze:\n",
    "# for i, layer in enumerate(resnet_model.layers):\n",
    "#    print(i, layer.name)\n",
    "\n",
    "model1 = Sequential()\n",
    "model1.add(mobilenet_model)\n",
    "\n",
    "model1.add(Flatten())\n",
    "model1.add(BatchNormalization())\n",
    "\n",
    "model1.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2796d776",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22967 samples, validate on 5742 samples\n",
      "Epoch 1/10\n",
      "22967/22967 [==============================] - 341s 15ms/sample - loss: 0.8755 - accuracy: 0.7139 - val_loss: 20.7680 - val_accuracy: 0.0012\n",
      "Epoch 2/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.6875 - accuracy: 0.7749 - val_loss: 39.0862 - val_accuracy: 0.0000e+00\n",
      "Epoch 3/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.5117 - accuracy: 0.8340 - val_loss: 29.6083 - val_accuracy: 0.0023\n",
      "Epoch 4/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.4682 - accuracy: 0.8491 - val_loss: 46.3194 - val_accuracy: 0.0028\n",
      "Epoch 5/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.4308 - accuracy: 0.8645 - val_loss: 75.8165 - val_accuracy: 0.0000e+00\n",
      "Epoch 6/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.3573 - accuracy: 0.8902 - val_loss: 80.2613 - val_accuracy: 0.0000e+00\n",
      "Epoch 7/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.3408 - accuracy: 0.8982 - val_loss: 35.9675 - val_accuracy: 0.1801\n",
      "Epoch 8/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.3585 - accuracy: 0.8932 - val_loss: 50.0373 - val_accuracy: 0.2344\n",
      "Epoch 9/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.3473 - accuracy: 0.8998 - val_loss: 96.6071 - val_accuracy: 3.4831e-04\n",
      "Epoch 10/10\n",
      "22967/22967 [==============================] - 436s 19ms/sample - loss: 0.3462 - accuracy: 0.9033 - val_loss: 53.5345 - val_accuracy: 0.0688\n"
     ]
    }
   ],
   "source": [
    "# Couldn't use data augmentation due to memory limitation.\n",
    "# 3 + 10 epochs:\n",
    "\n",
    "model1.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "history1 = model1.fit(X_train, y_train, batch_size = 128, epochs=10, validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b5842782",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "model1.save_weights(os.path.join(Project_DIR, \"model1\", \"model1_mobileNet_FineTuned\"), save_format = \"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "032211b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR! Session/line number was not unique in database. History logging moved to new session 274\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x1e04bfec630>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.load_weights(os.path.join(Project_DIR, \"model1\", \"model1_mobileNet_FineTuned\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "70ad2ef6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7178, 7)\n",
      "ERROR! Session/line number was not unique in database. History logging moved to new session 275\n",
      "(7178, 7)\n",
      "Accuracy on the testing set is:  16.940651992198383\n"
     ]
    }
   ],
   "source": [
    "# Accuracy of model1:\n",
    "# After training for 1.5 hrs: 16.94\n",
    "\n",
    "# 16.94 => (halted)\n",
    "\n",
    "print(y_test.shape)\n",
    "predictions = model1.predict(X_test)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum = np.sum(classes == y_test)\n",
    "accuracy = (sum - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dc707f0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR! Session/line number was not unique in database. History logging moved to new session 258\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenetv2_1.00_128 (Model) (None, 4, 4, 1280)        2257984   \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 20480)             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 50)                1024050   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 50)                200       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 7)                 357       \n",
      "=================================================================\n",
      "Total params: 3,282,591\n",
      "Trainable params: 3,248,379\n",
      "Non-trainable params: 34,212\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model2 = Sequential()\n",
    "model2.add(mobilenet_model)\n",
    "\n",
    "model2.add(Flatten())\n",
    "\n",
    "model2.add(Dense(50, activation='relu'))\n",
    "model2.add(Dropout(0.20))\n",
    "model2.add(BatchNormalization())\n",
    "\n",
    "model2.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee15dcdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "history2 = model2.fit(X_train, y_train, batch_size = 128, epochs=4, validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd086b21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following are predictions of model2 on the testing set after training for 5 hours locally in breaks\n",
    "\n",
    "# Progress: 30 => 36 => 36.5 => 35.1 => 32.66 => 38.83 => 21.05\n",
    "\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "\n",
    "predictions = model2.predict(X_test)\n",
    "print(predictions.shape)\n",
    "\n",
    "# print(\"Predictions: \", predictions)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "\n",
    "print(classes.shape)\n",
    "print(np.sum(classes == y_test) + np.sum(classes != y_test))\n",
    "\n",
    "print(classes == y_test)\n",
    "sum = np.sum(classes == y_test)\n",
    "print(\"Sum is: \", sum)\n",
    "print(\"y_train.shape[0] = \", y_test.shape[0])\n",
    "\n",
    "# By derivation: accuracy = (sum - m * (n - 2)) / (2 * m), n = num of classes, m = y_test.shape[0]\n",
    "\n",
    "accuracy = (sum - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd729e51",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint: Save model2 and train later\n",
    "\n",
    "# ** REMEMBER: Always save and load models as follows if you want to resume training from where you left last**\n",
    "\n",
    "# model.save_weights('My_weights_tf',save_format='tf')\n",
    "# model.load_weights('My_weights_tf')\n",
    "\n",
    "# Previous training accuracy = 0.92\n",
    "\n",
    "model2.save_weights(os.path.join(Project_DIR, \"model2\", \"model2_mobilenetV2\"), save_format = \"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7909996a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Before loading the model, build the model again by running the appropriate cell with randomly initialized weights\n",
    "# As we only saved the model weights, not the architecture\n",
    "\n",
    "model2.load_weights(os.path.join(Project_DIR, \"model2\", \"model2_mobilenetV2\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b35b63a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 128, 128, 8)       224       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 64, 64, 8)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 64, 64, 8)         32        \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 64, 64, 16)        1168      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 32, 32, 16)        64        \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 32, 32, 32)        4640      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 16, 16, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 8, 8, 64)          256       \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 8, 128)         73856     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 4, 4, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 4, 4, 256)         295168    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 2, 2, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 2, 2, 256)         1024      \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 200)               205000    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 200)               800       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 160)               32160     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 160)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 160)               640       \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 120)               19320     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 7)                 847       \n",
      "=================================================================\n",
      "Total params: 654,335\n",
      "Trainable params: 652,607\n",
      "Non-trainable params: 1,728\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# MODEL3 (Custom):\n",
    "\n",
    "model3 = Sequential()\n",
    "\n",
    "model3.add(Conv2D(8, kernel_size = (3,3), activation = 'relu', padding = 'same', input_shape = (img_size, img_size, 3)))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Conv2D(16, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Conv2D(32, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Conv2D(64, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Conv2D(128, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Conv2D(256, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model3.add(MaxPool2D(pool_size=(2,2)))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Flatten())\n",
    "\n",
    "model3.add(Dense(200, activation='relu'))\n",
    "model3.add(Dropout(0.2))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Dense(160, activation='relu'))\n",
    "model3.add(Dropout(0.2))\n",
    "model3.add(BatchNormalization())\n",
    "\n",
    "model3.add(Dense(120, activation='relu'))\n",
    "\n",
    "model3.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model3.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cab89c31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22967 samples, validate on 5742 samples\n",
      "Epoch 1/20\n",
      "ERROR! Session/line number was not unique in database. History logging moved to new session 269\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0813 - accuracy: 0.9720 - val_loss: 13.8984 - val_accuracy: 0.1043\n",
      "Epoch 2/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0679 - accuracy: 0.9770 - val_loss: 14.9252 - val_accuracy: 0.1257\n",
      "Epoch 3/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0590 - accuracy: 0.9802 - val_loss: 15.2621 - val_accuracy: 0.1003\n",
      "Epoch 4/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0603 - accuracy: 0.9788 - val_loss: 15.0652 - val_accuracy: 0.1230\n",
      "Epoch 5/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0686 - accuracy: 0.9773 - val_loss: 14.0797 - val_accuracy: 0.1304\n",
      "Epoch 6/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0557 - accuracy: 0.9817 - val_loss: 14.6117 - val_accuracy: 0.1261\n",
      "Epoch 7/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0562 - accuracy: 0.9814 - val_loss: 14.6454 - val_accuracy: 0.1346\n",
      "Epoch 8/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0522 - accuracy: 0.9825 - val_loss: 14.9933 - val_accuracy: 0.1304\n",
      "Epoch 9/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0531 - accuracy: 0.9825 - val_loss: 16.1813 - val_accuracy: 0.1050\n",
      "Epoch 10/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0545 - accuracy: 0.9813 - val_loss: 15.4559 - val_accuracy: 0.1494\n",
      "Epoch 11/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0430 - accuracy: 0.9853 - val_loss: 17.2538 - val_accuracy: 0.1233\n",
      "Epoch 12/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0460 - accuracy: 0.9852 - val_loss: 16.3562 - val_accuracy: 0.1308\n",
      "Epoch 13/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0437 - accuracy: 0.9848 - val_loss: 16.3971 - val_accuracy: 0.1061\n",
      "Epoch 14/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0542 - accuracy: 0.9818 - val_loss: 16.3281 - val_accuracy: 0.1118\n",
      "Epoch 15/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0440 - accuracy: 0.9854 - val_loss: 16.2475 - val_accuracy: 0.1108\n",
      "Epoch 16/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0466 - accuracy: 0.9842 - val_loss: 16.8732 - val_accuracy: 0.1237\n",
      "Epoch 17/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0491 - accuracy: 0.9839 - val_loss: 15.2467 - val_accuracy: 0.1296\n",
      "Epoch 18/20\n",
      "22967/22967 [==============================] - 139s 6ms/sample - loss: 0.0438 - accuracy: 0.9848 - val_loss: 16.2200 - val_accuracy: 0.1214\n",
      "Epoch 19/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0391 - accuracy: 0.9869 - val_loss: 16.8725 - val_accuracy: 0.1050\n",
      "Epoch 20/20\n",
      "22967/22967 [==============================] - 138s 6ms/sample - loss: 0.0313 - accuracy: 0.9891 - val_loss: 16.8726 - val_accuracy: 0.1310\n"
     ]
    }
   ],
   "source": [
    "# 30 + 20 epochs:\n",
    "model3.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "history3 = model3.fit(X_train, y_train, batch_size = 128, epochs=20, validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6257a5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "model3.save_weights(os.path.join(Project_DIR, \"model3\", \"model3\"), save_format = \"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8ea086a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x21409e1bac8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "model3.load_weights(os.path.join(Project_DIR, \"model3\", \"model3\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ec0d1e7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmH0lEQVR4nO3de5xU9X3/8ddnZm8su1zkLqCgIgKKqCuS2KTkqhgVTdSiNBfTak1iEv01qbbpL7FN+mvzy6WXaENsamMajeItGoMkakRjFRWMgjcUjciC4AICy8JeZufTP75ndmeHWRhgZ2d3z/v5eMxjzvl+z5n5zJkz388533MZc3dERCS+EqUOQERESkuJQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCCRWzOwnZvatAqd908w+XOyYREpNiUBEJOaUCET6ITMrK3UMMnAoEUifE3XJfNXMVplZk5n9p5mNMbMHzKzRzB4ys+FZ059rZi+a2XYzW2Zm07LqTjKzZ6P5bgeqct7rbDN7Lpr3CTObWWCMHzOz35vZTjNbb2bX5dT/UfR626P6z0Tlg8zse2a2zsx2mNnjUdlcM6vPsxw+HA1fZ2Z3mtnPzGwn8Bkzm21mT0bv8baZXW9mFVnzzzCzB81sm5ltNrO/MbOxZrbbzEZkTXeKmTWYWXkhn10GHiUC6as+AXwEOBY4B3gA+BtgJGG9/RKAmR0L/By4ChgFLAF+aWYVUaP4C+C/gcOAO6LXJZr3ZOAm4C+AEcCPgPvMrLKA+JqATwHDgI8BnzOz86LXPSKK9wdRTLOA56L5vgucArw3iumvgHSBy2Q+cGf0nrcA7cDVhGXyHuBDwOejGGqBh4ClwOHAMcDD7r4JWAZclPW6fwrc5u5tBcYhA4wSgfRVP3D3ze6+Afgd8JS7/97dW4B7gJOi6f4E+JW7Pxg1ZN8FBhEa2jlAOfAv7t7m7ncCz2S9x2XAj9z9KXdvd/ebgZZovn1y92Xuvtrd0+6+ipCM/jiqXgg85O4/j953q7s/Z2YJ4LPAl919Q/SeT0SfqRBPuvsvovfc4+4r3X25u6fc/U1CIsvEcDawyd2/5+7N7t7o7k9FdTcTGn/MLAlcTEiWElNKBNJXbc4a3pNnvCYaPhxYl6lw9zSwHhgf1W3wrndWXJc1fCTwl1HXynYz2w5MjObbJzM7zcweibpUdgBXELbMiV7j9TyzjSR0TeWrK8T6nBiONbP7zWxT1F30/wqIAeBeYLqZHUXY69rh7k8fZEwyACgRSH+3kdCgA2BmRmgENwBvA+OjsowjsobXA//g7sOyHtXu/vMC3vdW4D5gorsPBRYBmfdZDxydZ54tQHM3dU1AddbnSBK6lbLl3ir4h8ArwBR3H0LoOttfDLh7M7CYsOfySbQ3EHtKBNLfLQY+ZmYfig52/iWhe+cJ4EkgBXzJzMrM7OPA7Kx5/wO4Itq6NzMbHB0Eri3gfWuBbe7ebGazgUuy6m4BPmxmF0XvO8LMZkV7KzcB3zezw80saWbviY5JvApURe9fDvwtsL9jFbXATmCXmR0HfC6r7n5grJldZWaVZlZrZqdl1f8U+AxwLvCzAj6vDGBKBNKvufsaQn/3Dwhb3OcA57h7q7u3Ah8nNHjvEo4n3J017wrCcYLro/q10bSF+Dzw92bWCHydkJAyr/sWcBYhKW0jHCg+Mar+CrCacKxiG/BtIOHuO6LX/DFhb6YJ6HIWUR5fISSgRkJSuz0rhkZCt885wCbgNeADWfX/QzhI/Wx0fEFizPTHNCLxZGa/BW519x+XOhYpLSUCkRgys1OBBwnHOBpLHY+UlrqGRGLGzG4mXGNwlZKAgPYIRERiT3sEIiIx1+9uXDVy5EifNGlSqcMQEelXVq5cucXdc69NAfphIpg0aRIrVqwodRgiIv2Kma3rrk5dQyIiMadEICISc0oEIiIxV7RjBGZ2E+FWuO+4+/F56g34V8Kl+LuBz7j7swfzXm1tbdTX19Pc3HwoIfcLVVVVTJgwgfJy/YeIiPSMYh4s/gnhHi4/7aZ+HjAlepxGuJPiad1Mu0/19fXU1tYyadIkut5ocmBxd7Zu3Up9fT2TJ08udTgiMkAUrWvI3R8j3FSrO/OBn3qwHBhmZuMO5r2am5sZMWLEgE4CAGbGiBEjYrHnIyK9p5THCMbT9Y826qOyvZjZ5Wa2wsxWNDQ05H2xgZ4EMuLyOUWk95TyOoJ8LVre+124+43AjQB1dXW6J4aI9DvptNOSStOaStOSaqclem5uS3cMd9anaWnLTBPVtaWpmzSc903Je03YISllIqgn/JNUxgTCv031O9u3b+fWW2/l85///AHNd9ZZZ3HrrbcybNiw4gQ2AKTTTirttKedVLrrf7xn7x3lblVk7zhZVm0yYZQljETi0Pes0mmnqTXFrpYUu5pTNLakaGwOw7ta2sJwS2Y8PPLe2qubUPIVJ8wwy3mOyhOJMFcit97CMkgYVJUnGVSRZFCe5+qKZEd9dVZdRTLRZVm7O3va2tnVkmJ3S3huakmxu7VzuKm1PTy3pGhqTdHUEsbNoCyR6PgeypJGMpGgLGEdZcmkUZ41TTIZPScSJCwsF+v4XEC0DDKfM1OeiAY6pgfS7qTdaU9DuzvpaN0KZZ5VFuo9p7w1laa1Pd3RQLd2PHc24q1d6to7xlPpQ9+G/dzcowdcIrgPuNLMbiMcJN7h7m+XMJ4D5tFK1bB1Gzfc8O9c+ueXk/bOla2trR1LJKLxUJ69Uv7k9rtpw9iyq2Wvldu6rNxdx9va07zesIvyRIKypFGeTFCeNMqS4QdVkUzss6FLtafZ1ZJi554UO5vb2LmnLTw3p6LhVEdZY1bZrpY23OnSsGTiyv3BZRohcsracxr29vbs8azyaLxY90QMDVJofMoTiazGxro0VMlMWdJImIXGLqthL0R1RZLaqjIGV5Tt9b10d9PH/LvGoTyzfrmDR+uVd6x3YcJ0TnnmubmtnQNtjxIGg8pDkmhNpWlqTRX8GpVlCQZXljG4MsngitDcZL7ntvZ01+89Gm+Lxtt7oOE8VGaQtLDhkDSjoixBRVmCyui5IpmgsjxJZTJBTWUZFdUJKstDeZgu2WX6yrIklWVhmo7hsug1MsNZ82RPt7/f9aEo5umjPwfmAiPNrB74BlAO4O6LgCWEU0fXEk4fvbRYsXTHvXNLIJW18rWnuymLtgqyG3aAv7r6K7z++uucdNJJlJWVM2jwYEaNHsOal17gnt8u56o/W8imtzfQ2tLCws9ewScWfhqAee+Zya2/eoTdTU184VMXctKpc3hu5dOMHjOOf/3PW6gaNChv3Jt3tnDZzx7d52dLGJQlE5QnjPKyBGXR1lRma21/aqvKGFJVHp4HlTN+2CBqKmtImOHRsguNUudwaKg6GyiPGiSPKt3paFSTiRBb5/jeDXBmKzBTn4wSZfjusr7HnGaza13X8rQ7qXanPR220FLpruNdE1JU3t75/U8YnqSmsoyaynJqqsqorSyjtqqMmqoyajLDUV2YroxkkX68B8PdaW1P09yaZndbij2t7exubae5rZ09bVnDUfmeaHhPVF9ZFhq8wZVlDK5IRo18SHKDK5NZdWVUVyYpTx78YUjP+m2m0k57e5QA6VznMutZJkl69jpI2GsjqzyR1agnEqGRTyY6y5KJkPDDc3yOyfW721DX1dV57r2GXn75ZaZNmwbA3/3yRV7auHOv+dIedus6G6zut8Y6RFu+x4yu4YsfPKZjJUkYJDqGjfq33uSSCz/O8pXP8T+/e5QLz5/Pimef56ijJ5MwY/u77zJyxGE0Nzdz6qmnsmzZMg4bMYKjJk9m+dNP09i4i2lTj+XJ5U8x88RZXHLxAj529tksuGRhl0Y1s3K//uoa1vtw2trDVlRb2mlLpUml01FZ2NpqixqxVHua1vaQxGqiBn7IoDJqq8oZEjX02WV9rfESkUNnZivdvS5fXb+76dzByjSiBlgi6jfu6LLo7LogazxjWHU5R42q6fa1mwZXkkwYQweVU11RxuzZs5k29ZiO+huu/wH33HMPAOvXr2ft2rXMGTkSCP2l5ckEkydP5tS6UwCYfWodG+vXU12R/+sZVJFk/rS8J1iJiBywAZcIvnHOjFKHwODBgzuGly1bxkMPPcSTTz5JdXU1c+fOzXsdQGVlZcdwMplkz549vRKriIjuNdQDamtraWzM/49/O3bsYPjw4VRXV/PKK6+wfPnyXo5ORGTfBtweQSmMGDGC008/neOPP55BgwYxZsyYjrozzzyTRYsWMXPmTKZOncqcOXNKGKmIyN4G3MHiOIjb5xWRQ7evg8XqGhIRiTklAhGRmFMiEBGJOSUCEZGYUyIQEYk5JQIRkZhTIiiBmprub1chItLblAhERGJOVxb3gGuuuYYjjzyy449prrvuOsyMxx57jHfffZe2tja+9a1vMX/+/BJHKiKyt4GXCB64Fjat7tnXHHsCzPunbqsXLFjAVVdd1ZEIFi9ezNKlS7n66qsZMmQIW7ZsYc6cOZx77rmxub+5iPQfAy8RlMBJJ53EO++8w8aNG2loaGD48OGMGzeOq6++mscee4xEIsGGDRvYvHkzY8eOLXW4IiJdDLxEsI8t92K64IILuPPOO9m0aRMLFizglltuoaGhgZUrV1JeXs6kSZPy3n5aRKTUBl4iKJEFCxZw2WWXsWXLFh599FEWL17M6NGjKS8v55FHHmHdunWlDlFEJC8lgh4yY8YMGhsbGT9+POPGjWPhwoWcc8451NXVMWvWLI477rhShygikpcSQQ9avbrzIPXIkSN58skn8063a9eu3gpJRGS/dB2BiEjMKRGIiMTcgEkE/e2f1g5WXD6niPSeAZEIqqqq2Lp164BvJN2drVu3UlVVVepQRGQAGRAHiydMmEB9fT0NDQ2lDqXoqqqqmDBhQqnDEJEBZEAkgvLyciZPnlzqMERE+qUB0TUkIiIHT4lARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGKuqInAzM40szVmttbMrs1TP9TMfmlmz5vZi2Z2aTHjERGRvRUtEZhZErgBmAdMBy42s+k5k30BeMndTwTmAt8zs4pixSQiInsr5h7BbGCtu7/h7q3AbcD8nGkcqDUzA2qAbUCqiDGJiEiOYiaC8cD6rPH6qCzb9cA0YCOwGviyu6dzX8jMLjezFWa2Ig53GBUR6U3FTASWpyz3DwPOAJ4DDgdmAdeb2ZC9ZnK/0d3r3L1u1KhRPR2niEisFTMR1AMTs8YnELb8s10K3O3BWuAPwHFFjElERHIUMxE8A0wxs8nRAeAFwH0507wFfAjAzMYAU4E3ihiTiIjkKNof07h7ysyuBH4NJIGb3P1FM7siql8EfBP4iZmtJnQlXePuW4oVk4iI7K2o/1Dm7kuAJTlli7KGNwIfLWYMIiKyb7qyWEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQEQk5gpKBGZ2l5l9zMyUOEREBphCG/YfApcAr5nZP5nZcUWMSUREelFBicDdH3L3hcDJwJvAg2b2hJldamblxQxQRESKq+CuHjMbAXwG+HPg98C/EhLDg0WJTEREekVZIROZ2d3AccB/A+e4+9tR1e1mtqJYwYmISPEVukdwvbtPd/d/zEoCALh7XXczmdmZZrbGzNaa2bXdTDPXzJ4zsxfN7NEDiF1ERHpAoYlgmpkNy4yY2XAz+/y+ZjCzJHADMA+YDlxsZtNzphkG/DtwrrvPAC4sPHQREekJhSaCy9x9e2bE3d8FLtvPPLOBte7+hru3ArcB83OmuQS4293fil73nQLjERGRHlJoIkiYmWVGoq39iv3MMx5YnzVeH5VlOxYYbmbLzGylmX0q3wuZ2eVmtsLMVjQ0NBQYsoiIFKKgg8XAr4HFZrYIcOAKYOl+5rE8ZZ7n/U8BPgQMAp40s+Xu/mqXmdxvBG4EqKury30NERE5BIUmgmuAvwA+R2jgfwP8eD/z1AMTs8YnABvzTLPF3ZuAJjN7DDgReBUREekVBSUCd08Tri7+4QG89jPAFDObDGwAFhCOCWS7F7jezMoIXU2nAf98AO8hIiKHqNDrCKYA/0g4+6cqU+7uR3U3j7unzOxKQrdSErjJ3V80syui+kXu/rKZLQVWAWngx+7+wkF/GhEROWCFdg39F/ANwtb6B4BLyX8MoAt3XwIsySlblDP+HeA7BcYhIiI9rNCzhga5+8OAufs6d78O+GDxwhIRkd5S6B5Bc3QL6tei7p4NwOjihSUiIr2l0D2Cq4Bq4EuE0z3/FPh0kWISEZFetN89gujisYvc/avALsLxARERGSD2u0fg7u3AKdlXFouIyMBR6DGC3wP3mtkdQFOm0N3vLkpUIiLSawpNBIcBW+l6ppADSgQiIv1coVcW67iAiMgAVeiVxf/F3jeMw90/2+MRiYhIryq0a+j+rOEq4Hz2voGciIj0Q4V2Dd2VPW5mPwceKkpEIiLSqwq9oCzXFOCIngxERERKo9BjBI10PUawifAfBSIi0s8V2jVUW+xARESkNArqGjKz881saNb4MDM7r2hRiYhIryn0GME33H1HZsTdtxP+n0BERPq5QhNBvukKPfVURET6sEITwQoz+76ZHW1mR5nZPwMrixmYiIj0jkITwReBVuB2YDGwB/hCsYISEZHeU+hZQ03AtUWORURESqDQs4YeNLNhWePDzezXRYtKRER6TaFdQyOjM4UAcPd30X8Wi4gMCIUmgrSZddxSwswmkedupCIi0v8Uegro14DHzezRaPz9wOXFCUlERHpToQeLl5pZHaHxfw64l3DmkIiI9HOF3nTuz4EvAxMIiWAO8CRd/7pSRET6oUKPEXwZOBVY5+4fAE4CGooWlYiI9JpCE0GzuzcDmFmlu78CTC1eWCIi0lsKPVhcH11H8AvgQTN7F/1VpYjIgFDoweLzo8HrzOwRYCiwtGhRiYhIrzngO4i6+6P7n0pERPqLg/3PYhERGSCUCEREYk6JQEQk5pQIRERirqiJwMzONLM1ZrbWzLr9PwMzO9XM2s3sgmLGIyIieytaIjCzJHADMA+YDlxsZtO7me7bgP7fQESkBIq5RzAbWOvub7h7K3AbMD/PdF8E7gLeKWIsIiLSjWImgvHA+qzx+qisg5mNB84HFu3rhczscjNbYWYrGhp0iyMRkZ5UzERgecpy/8zmX4Br3L19Xy/k7je6e527140aNaqn4hMREQ7iyuIDUA9MzBqfwN73J6oDbjMzgJHAWWaWcvdfFDEuERHJUsxE8AwwxcwmAxuABcAl2RO4++TMsJn9BLhfSUBEpHcVLRG4e8rMriScDZQEbnL3F83siqh+n8cFRESkdxRzjwB3XwIsySnLmwDc/TPFjEVERPLTlcUiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEIiIxp0QgIhJzRU0EZnamma0xs7Vmdm2e+oVmtip6PGFmJxYzHhER2VvREoGZJYEbgHnAdOBiM5ueM9kfgD9295nAN4EbixWPiIjkV8w9gtnAWnd/w91bgduA+dkTuPsT7v5uNLocmFDEeEREJI9iJoLxwPqs8fqorDt/BjyQr8LMLjezFWa2oqGhoQdDFBGRYiYCy1PmeSc0+wAhEVyTr97db3T3OnevGzVqVA+GKCIiZUV87XpgYtb4BGBj7kRmNhP4MTDP3bcWMR4REcmjmHsEzwBTzGyymVUAC4D7sicwsyOAu4FPuvurRYxFRES6UbQ9AndPmdmVwK+BJHCTu79oZldE9YuArwMjgH83M4CUu9cVKyYREdmbuefttu+z6urqfMWKFaUOQ0SkXzGzld1taOvKYhGRmFMiEBGJOSUCEZGYUyIQEYk5JQKRQrz1FCz+NDz2Xdj+VqmjEelRxbygTKT/270NHroOnr0ZKofCS7+A334TjvwjmHkRzDgPqoaWOEiRQ6NEIJKPOzx/G/zma7BnO7z3i/DH18LuLbDqDlh1G/zyS7DkqzB1Hpy4AI75MCTLSx25yAHTdQRSPO0p2LUJtq+HHfVhuKwqbEFX1kLlEKga0vU5kSx11NCwBu7/P7DucZgwG87+Zxh7fNdp3GHjs/D87fDCnbB7K1SPgOM/ATP/BMafApbvdlsipbGv6wiUCAaqVAs0boLGt2HnxjC8ewuUD4KqYVHjOzQ0wFVDO8craiBR4KGjll2wI2rkd6zvbPAz4zs3grcfWNwVNSGWytq9k0T1YWGr+4j3Fh7jgWjdDb/7LvzPv0HFYPjI38FJn9r/e7W3wdqHw17Cmgcg1QwjjgkJYeZFMHzSgcWRTofE0rgRdr4dvsPGt0PZxNPCHkhl7UF/zNhIt8Oed6FpCzQ1hOGqITBkAgwdH34L/UE6DeuXw6rb4agPhO7Ig6BEcKjaU+GL2PQCHP0BGDW1d98/W3Yj0bgpauSjhiK30chlCfD0ft7Auk8S5VWw652owV8PzdtzZk3CkPEwbCIMnRA9JkaPCVA7NiSolp3QvDM8Zw/vq6x5Z/ghp9ug9nA44RNwwkUw9oSe2fJ+9Tew5CuwfR2ceAl89JsweOSBv07zDnjpvvCjffN3oeyI94SkMOM8SJTnfF/Z32OUuBs3hc/ZhYXk1Lor7FUde0bY+5jy0f7ToB2q9lTYmGnakvW8tfvx3dvo5obHQfWIsL4OnRA9j+9MEkPGw5DDS9vV1/BqWI9WLw4nKJRXw9y/htO/dFAvp0RwMHZvC1t5ry6FtQ+GH3jG2BNCI3T8J8JKU0ypFnjjUXjlfnhjWWgw8jUSNaNDQ1t7OAwZB7VZj8z4oOFdG+LmHdCyI2s4eu5uvK0JasZ2NvLDshr5oRPD+xeza6e1KWxxr74zfCfpFIycCidcCCdcAIdNPvDX3LkRll4LL90LI4+Fj30fJr+vZ+Ldvj78iJ+/HbasIdyZPc/vraI2+o6i7692bGiEsr+/mjEh0dY/DS/cBS/eE7ZyK2pg6llhXTz6g1BW0TOxl5J7SIibX4TNL0TPL4VlmE7lmcHC3mL1yJC8q0dEzznjg4aH9XjHBthZHz1v6BzP/o1nXrdmTGdiGDox/PYnzobDjipO11/j5vD9rrod3n4ubLwd/cGwITH1LKisOeiXViIohHvoG351Kbz667AH4GkYPAqmnBG2wMYeH+pW3wEbVgIGR54OMy+EaeeGlbEnNO+A1x4Mjf9rD4atwIrasDcy4ui9G/ua0fE7SLl7WziDZ9Ud8NYToWzCqSEpzPg41OznfyvaU/D0jfDIP4TG5f1fhfd+qTgNqXv4Ua9ZGvaqOhr4qNE/mG6edDu8+XhoNF66N+ydVQ2DaeeEpDDpfZDsB+eCtDbBOy9Hjf2LnY1/9t7m0IkwZgaMnh4a5Y4GfmRnA98TGyAtu6LEUN81QezIKmvbHaatHhm66SbOhiPmwLhZ4bs9GK1N8MqvQuP/+m9DuzNuVmj8j/8E1I459M+GEkH3Ui3hx/Tqr0MC2L4ulI89AY49MzwOPzl/H/HW18OW6erFsHVt2OWf8pHQEB17JlRUH1gsjZtgzRJ4+X74w2Nhq3/wqLAVMO0cmPx+KKs89M88EG1fHw7Yrr4zNCKWhKPmhu9i2tl7N7T1K+H+L8Om1XDMR+Cs7xzc3kRfkWoNe4sv3BUalNbGsO5MPy80JBNPO/BjKu5hg2T31qzHttBIJcqiRyJruCws90QyGs96tmgYYNsb8M5LnVv62/5Ax15SRU1o7MfM6HyMng6DhvXcsjoU6XTYK1n/FKx/Gt5aDtteD3WJcjh8VmdymHhaSPLdaU/BH5bBqsXhN9/WBEOPCMeUZl5UlO5nJYJsu96B134TGv7XH+nscz1qbtjqn3LGgXX3ZLb2VkcN0a5NYYWedk7orpg8t/sts62vw8u/DD/e+mcAh+GTQ+N13NlhC7cvnEXTn2x+Keyxrb4TdrwVvtup80JX3oRTYdk/woqbwo903rfDntxAOrunbU/Yi3zhrrCBk9oTujVmnA/T54fjCZm+9N3bor70rVllWY+83TA9wcKe7ejpMOb4zkZ/2JHFOQmgmJq2hKSQSQ4bnw0nCwAMOwImzulMDKOnhwS4anHYcNm1ORx7m3F+2PqfOKeon1+JAEI/+8N/F3XpEH4cx54Rtt4nve/At+Dzyeyur74jHDBs2RG2zGacHxqi8aeEpPHKr0K3T8MrYb5xJ4aG/7izYfS0gdUwlYp7+HGuviP0p2cOnlsCTrsiHHSrGlLaGIutpTF0R71wF6x9KM+xJQALXSsd/eojOh+549WHheWXbo8eqfDwzHA3z9n1w4+EUceFA98DUaoVNq2KEsNT4Yr0XZtCXbIC2lvD87FnhMZ/ykd7bU9fiQBCtl7611GXzxk9d7ZJd9qawwHN1XeEH2N7C5QPDruAlgjHFo47G447K2w5SPG0t4W9v3WPh66ScSeWOqLet+fdsAwSydC/nX0AVXudxeMezvjJ7C2MnBK67HrqeOIBUCIoteYdoR9w/fKw+3fsmTB4RKmjEpEY2Vci6AenFQwAVUPhpIXhISLSx/SzIzMiItLTlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIRERiTolARCTmlAhERGKu311ZbGYNwLqDnH0ksKUHw+lpfT0+6PsxKr5Do/gOTV+O70h3z3t/9n6XCA6Fma3o7hLrvqCvxwd9P0bFd2gU36Hp6/F1R11DIiIxp0QgIhJzcUsEN5Y6gP3o6/FB349R8R0axXdo+np8ecXqGIGIiOwtbnsEIiKSQ4lARCTmBmQiMLMzzWyNma01s2vz1JuZ/VtUv8rMTu7F2Caa2SNm9rKZvWhmX84zzVwz22Fmz0WPr/dWfNH7v2lmq6P33uvv4Eq8/KZmLZfnzGynmV2VM02vLz8zu8nM3jGzF7LKDjOzB83steh5eDfz7nN9LWJ83zGzV6Lv8B4zG9bNvPtcH4oY33VmtiHrezyrm3lLtfxuz4rtTTN7rpt5i778Dpm7D6gHkAReB44CKoDngek505wFPAAYMAd4qhfjGwecHA3XAq/miW8ucH8Jl+GbwMh91Jds+eX5rjcRLpQp6fID3g+cDLyQVfb/gWuj4WuBb3fzGfa5vhYxvo8CZdHwt/PFV8j6UMT4rgO+UsA6UJLll1P/PeDrpVp+h/oYiHsEs4G17v6Gu7cCtwHzc6aZD/zUg+XAMDMb1xvBufvb7v5sNNwIvAyM74337kElW345PgS87u4He6V5j3H3x4BtOcXzgZuj4ZuB8/LMWsj6WpT43P037p6KRpcDE3r6fQvVzfIrRMmWX4aZGXAR8POeft/eMhATwXhgfdZ4PXs3tIVMU3RmNgk4CXgqT/V7zOx5M3vAzGb0bmQ48BszW2lml+ep7xPLD1hA9z++Ui6/jDHu/jaEDQBgdJ5p+sqy/CxhLy+f/a0PxXRl1HV1Uzdda31h+b0P2Ozur3VTX8rlV5CBmAgsT1nuObKFTFNUZlYD3AVc5e47c6qfJXR3nAj8APhFb8YGnO7uJwPzgC+Y2ftz6vvC8qsAzgXuyFNd6uV3IPrCsvwakAJu6WaS/a0PxfJD4GhgFvA2ofslV8mXH3Ax+94bKNXyK9hATAT1wMSs8QnAxoOYpmjMrJyQBG5x97tz6919p7vvioaXAOVmNrK34nP3jdHzO8A9hN3vbCVdfpF5wLPuvjm3otTLL8vmTJdZ9PxOnmlKvS5+GjgbWOhRh3auAtaHonD3ze7e7u5p4D+6ed9SL78y4OPA7d1NU6rldyAGYiJ4BphiZpOjrcYFwH0509wHfCo6+2UOsCOzC19sUX/ifwIvu/v3u5lmbDQdZjab8D1t7aX4BptZbWaYcEDxhZzJSrb8snS7FVbK5ZfjPuDT0fCngXvzTFPI+loUZnYmcA1wrrvv7maaQtaHYsWXfdzp/G7et2TLL/Jh4BV3r89XWcrld0BKfbS6GA/CWS2vEs4m+FpUdgVwRTRswA1R/Wqgrhdj+yPCrusq4LnocVZOfFcCLxLOgFgOvLcX4zsqet/noxj61PKL3r+a0LAPzSor6fIjJKW3gTbCVuqfASOAh4HXoufDomkPB5bsa33tpfjWEvrXM+vhotz4ulsfeim+/47Wr1WExn1cX1p+UflPMutd1rS9vvwO9aFbTIiIxNxA7BoSEZEDoEQgIhJzSgQiIjGnRCAiEnNKBCIiMadEINKLLNwZ9f5SxyGSTYlARCTmlAhE8jCzPzWzp6N7yP/IzJJmtsvMvmdmz5rZw2Y2Kpp2lpktz7qv//Co/Bgzeyi6+d2zZnZ09PI1Znanhf8CuCVzFbRIqSgRiOQws2nAnxBuFjYLaAcWAoMJ9zc6GXgU+EY0y0+Ba9x9JuFK2Ez5LcANHm5+917ClakQ7jh7FTCdcOXp6UX+SCL7VFbqAET6oA8BpwDPRBvrgwg3jEvTeXOxnwF3m9lQYJi7PxqV3wzcEd1fZry73wPg7s0A0es97dG9aaJ/tZoEPF70TyXSDSUCkb0ZcLO7/3WXQrP/mzPdvu7Psq/unpas4Xb0O5QSU9eQyN4eBi4ws9HQ8d/DRxJ+LxdE01wCPO7uO4B3zex9UfkngUc9/MdEvZmdF71GpZlV9+aHECmUtkREcrj7S2b2t4R/lUoQ7jj5BaAJmGFmK4EdhOMIEG4xvShq6N8ALo3KPwn8yMz+PnqNC3vxY4gUTHcfFSmQme1y95pSxyHS09Q1JCISc9ojEBGJOe0RiIjEnBKBiEjMKRGIiMScEoGISMwpEYiIxNz/Aj3nhO8vAv4JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history3.history['accuracy'])\n",
    "plt.plot(history3.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5bc4aa03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAocklEQVR4nO3deZwU1bn/8c8zw7CDrCqLCDGGuCHCiLheFTWACyYiYtwX0ESvem8WjSbG3N9NYvbElWAkbgR3FBUV930DBEVRQQOXAYQBZV+HeX5/nJqZpqkeZpjprmH6+369+tXVVaeqnj5T008tp06ZuyMiIpKuIOkARESkYVKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCFSD8zsLjP73xqWnWdmx9V1OSLZpgQhIiKxlCBERCSWEoTkjejUzk/M7AMzW2tmd5rZbmb2tJmtNrPnzax9SvlTzOwjM1thZi+b2T4p0w4ys+nRfA8AzdPWdZKZzYjmfdPM+uxgzKPMbK6ZfWVmk8ysazTezOwvZrbUzFZG32n/aNpQM/s4im2hmf14hypM8p4ShOSb04DjgW8BJwNPA9cCnQj/D1cAmNm3gAnAVUBnYDLwhJk1NbOmwGPAvUAH4KFouUTz9gPGAZcAHYG/A5PMrFltAjWzY4HfAiOALsB84P5o8gnAUdH3aAecASyPpt0JXOLubYD9gRdrs16RCkoQkm9udvcl7r4QeA14x93fd/eNwETgoKjcGcBT7v6cu28G/gi0AA4DBgJFwF/dfbO7Pwy8l7KOUcDf3f0dd9/i7ncDG6P5auMsYJy7T4/i+xlwqJn1BDYDbYBvA+bus919cTTfZmBfM2vr7l+7+/RarlcEUIKQ/LMkZXh9zOfW0XBXwh47AO5eDiwAukXTFvrWPV3OTxneE/hRdHpphZmtAPaI5quN9BjWEI4Surn7i8AtwK3AEjMba2Zto6KnAUOB+Wb2ipkdWsv1igBKECKZLCL80APhnD/hR34hsBjoFo2r0CNleAHwa3dvl/Jq6e4T6hhDK8Ipq4UA7n6Tu/cH9iOcavpJNP49dx8G7Eo4FfZgLdcrAihBiGTyIHCimQ0ysyLgR4TTRG8CbwFlwBVm1sTMvgcMSJn3DuBSMzskupjcysxONLM2tYzhX8AFZtY3un7xG8IpsXlmdnC0/CJgLbAB2BJdIznLzHaJTo2tArbUoR4kjylBiMRw90+Bs4GbgWWEC9onu/smd98EfA84H/iacL3i0ZR5pxKuQ9wSTZ8bla1tDC8AvwAeIRy17AWMjCa3JSSirwmnoZYTrpMAnAPMM7NVwKXR9xCpNdMDg0REJI6OIEREJJYShIiIxFKCEBGRWEoQIiISq0nSAdSnTp06ec+ePZMOQ0RkpzFt2rRl7t45blqjShA9e/Zk6tSpSYchIrLTMLP5mabpFJOIiMRSghARkVhZO8VkZuOAk4Cl7l7RT/0DQO+oSDtghbv3jZl3HrCa0EVAmbsXZytOERGJl81rEHcRuhq4p2KEu59RMWxmfwJWVjP/Me6+rK5BbN68mZKSEjZs2FDXRTVozZs3p3v37hQVFSUdiog0EllLEO7+atRv/TaiXjBHAMdma/0VSkpKaNOmDT179mTrzjcbD3dn+fLllJSU0KtXr6TDEZFGIqlrEEcCS9x9TobpDkwxs2lmNrq6BZnZaDObamZTS0tLt5m+YcMGOnbs2GiTA4CZ0bFjx0Z/lCQiuZVUgjiT8DjHTA53937AEOAyMzsqU0F3H+vuxe5e3LlzbFPeRp0cKuTDdxSR3Mp5gjCzJoSukh/IVMbdF0XvSwmPgRyQqazITu/fr8Kc55KOQmQbSRxBHAd84u4lcROjh6u0qRgmPJx9Vg7jq1crVqzgtttuq/V8Q4cOZcWKFfUfkDQsi96H+4bD+OHwwNmw+sukIxKplLUEYWYTCE/e6m1mJWZ2UTRpJGmnl8ysq5lNjj7uBrxuZjOBdwkPjn8mW3FmW6YEsWVL9Q/5mjx5Mu3atctSVNIgrPsKHjgXWnWGo68NRxG3DIBpd4Oe0yINQDZbMZ2ZYfz5MeMWER6yjrt/ARyYrbhy7ZprruHzzz+nb9++FBUV0bp1a7p06cKMGTP4+OOPOfXUU1mwYAEbNmzgyiuvZPTocE2+otuQNWvWMGTIEI444gjefPNNunXrxuOPP06LFi0S/mZSJ+Xl8OgoWPMlXPAMdO8PBwyHSVfAE1fAhw/ByX+DjnslHanksUbVF9P2/OqJj/h40ap6Xea+Xdvyy5P3yzj9xhtvZNasWcyYMYOXX36ZE088kVmzZlU2Rx03bhwdOnRg/fr1HHzwwZx22ml07Nhxq2XMmTOHCRMmcMcddzBixAgeeeQRzj5bT5Hcqb36e5j7PJz455AcICSD856A9++BKb+A2w+Do38Gh14OhXn1r1o7ZRuhfAs0bZl0JDtmy2ZYtbBuR40FhdCuR/3FFNFWl2MDBgzY6l6Fm266iYkTJwKwYMEC5syZs02C6NWrF3379gWgf//+zJs3L1fhSjbMeR5evhEOPBOKL9x6WkEB9D8f9v4OTP4xPP9LmPUIDLsFujSaA+v6saYU3rsD3vsHNG0FFz0PbXZLOqqa27g6nE5861ZYvahuy2q1K/wk010DOy6vEkR1e/q50qpVq8rhl19+meeff5633nqLli1bcvTRR8fey9CsWbPK4cLCQtavX5+TWCULvp4Pj14Mu+0Xjh4yNU9u2wXOuA9mT4Knfgxjj4HD/hOOvgaK8vz04tJP4K1b4IMHYctG+ObxMP8NmDASzn+q4R9JrF0G74yBd++ADSug55Fw9NVQ2Gy7s2bUpA7zVrfYrCxVKrVp04bVq1fHTlu5ciXt27enZcuWfPLJJ7z99ts5jk5yavMGePDccP1hxD3b/yEzg32HQa+jYMrP4Y2/hoRx8k3Q68ichNxguMO/X4E3b4G5z0GT5nDQWTDwMuj0TfhkMtz//XBdZ8S94Uisofl6fkhs0++FsvXw7ZPgiP+C7g23qzkliCzr2LEjhx9+OPvvvz8tWrRgt92qDoEHDx7MmDFj6NOnD71792bgwIEJRipZ9/RPYfEMGDmhdhefW7SHYbfCAafDE1fC3SdBv/Pg+P+BFu2yFW3DULYpnGJ761ZY8mFo8XXMdVB8EbRKORX77aEw+LfwzDXw/PVwwv8mF3O6JR/B638N38MKoM8ZcPgV0Ln3dmdNmnkjak5XXFzs6Q8Mmj17Nvvss09CEeVWPn3Xnc7798Hjl8ER/w3H/XLHl7NpHbz8m/CD2WpXOPGPsM/J9RdnQ7H+a5j6T3h3LKxeDJ33gUMvC0myqHn8PO4w+SfhusSJf4aDL4ovlwvu8H9vwet/gTlToKgVFF8AA38Iu3RLLq4YZjYtU4/ZOoIQybbFM+GpH4VTRcdcV7dlNW0Z9o73Pw0e/89wc90+p8DAH4SWPGUbw3n5sg1huOK1JWW4bANs2RSVid6tAJq1hqbRq1nrcOG3aZsMw1G5+m5d9dUX8PbtIaFuXgffOCZcoN9rUObrNRXMYPCNsGJ+SBTt94RvHle/8W1PeTl89kw4HbjgHWjZEY75eUhWLTvkNpZ6oAQhkk3rv4YHzoEWHeC0cfX3g9r1IBj9Erx5c2gRNXtSDWe0cP6+SbOqV2Ez8HLYtAY2rQ3vNdWkeVVCadYWmu8SvbeFZm1Shived9l2WtPWsOBdeOtmmP0kFDSBPiPC3vbu+9euXgqbwPBxMG4IPHg+XPRsaBCQbVs2w4cPh8RQ+gns0gOG/AEOOrvhXzSvhhKEyLqvwo9S87b1u9zycnj0Eli1CC54GlrHdya5wwqL4Mj/DjfYlX4a/eA3h8KmURKI3gtTkkFBk+3viZeXw+a1IVlsXBMljjUxw2th0+qq8RtWwcZV8PW88L5xVWjK6eXVr88KQpnm7cL3GTAa2uy+4/XSrA18/wH4xyAYPwJGvVC35VXHHWbeDy/+L6wqgV33he/dAft9N/x9dnJKEJLf5r8VWr+Ul4VmpAN/EH5g6sPrf4I5z8LQP8IeB9fPMuO061G/N0kVFER7+G2grlXhnpI8VoeksWEVbFy59bi2XcPF26attr/MmtilW0gS44bAv86ACybX37IrrF0WGg188iR0K4aT/gJ7H7/9BLwTUYKQ/DXzAZh0OeyyR2hR8tKvQ/v0I/4LDr64bvcbfP4ivPjrcFH14IvrL+adjVlVssm1LgeG0033nwmPjIIz7g13HNeHz6aERgcbVsDx/y9cQK+vZTcgDbCxsEiWucNLv4GJo2GPQ+Di5+HMCXDxi7B7n3DPwU0HhTt0yzbVfvkrFsDDF8Gu+4T+lBrRHuVOp/fgcOH606fguevrvrxNa+HJ/4J/nQ6tOsGol0KT1UaYHEAJosFp3bp10iE0bps3wCMXwyu/g75nwdmPVrUu6d4fzn0s3I3bbs/Q8uiWYpjxr9BCqCbKNsJD54WLliPurf/TGlJ7h1wCAy4JN6m9948dX07JNBhzZGh+e9h/huRQ24voOxklCMkfa5fBPafArIdh0C/DzWdNmm5brucRcOEzcNbD4Ua0x34Atw2EjyaGC7jVeeZnsHAanHpbuMNXGobBv4VvDQ7NX2v7cKYtZaGl2J3Hhx2A854ITY0z3Y/RiChBZNnVV1+91fMgbrjhBn71q18xaNAg+vXrxwEHHMDjjz+eYIR5YukncMex4Z6E0+8OrWWqO/VjFi44jn4ldIuBwUPnw9j/COef424wnXk/TL0TDrsC9j0lW99EdkRBIZx2J+y2f/g7fvlhzeZb/jmMOwFe/m1oLfaDN/Kqm5P8upP66WtqvmHU1O4HwJAbM05+//33ueqqq3jllVcA2HfffXnmmWdo164dbdu2ZdmyZQwcOJA5c+ZgZrRu3Zo1a2rRDj2F7qTO4POX4MHzQjPPM++v6l67Nsq3hGc0vPzb0Ixzj0Pg2F9U/Vh8OQv+cVzoV+ecx9Q9d0O1ahHcEd10d/ELoVPEOO4w7Z/w7HWh2fBJf4H9v5fbWHOkujupdQSRZQcddBBLly5l0aJFzJw5k/bt29OlSxeuvfZa+vTpw3HHHcfChQtZsmRJ0qE2TtPugvtOC80eR72wY8kBwh7ogSPh8qnhx2LFgtAn0j3DQgJ64Oxwk9jwerwZTupf266h+ev6FTDhjHDROd3qJfCvEeFi9B6HwA/farTJYXvya0uuZk8/m4YPH87DDz/Ml19+yciRIxk/fjylpaVMmzaNoqIievbsGdvNt9RB+ZbwLIU3bw7dLQz/Z/3cCFdYFJ7hcOCZMHUcvPZnuPfUcAPa+U9B613rvg7Jri594PR/hu7BH7k4dKte0Qpp9pPhiX6b1sKQ38PBoxpmz7A5kl8JIiEjR45k1KhRLFu2jFdeeYUHH3yQXXfdlaKiIl566SXmz5+fdIiNy6a1od37p0+Ff/DBN9b/Xn1Ri9D2vd95IVG06wE91BvvTuNb3wkJYPKPQ7PmY64Np6Bn3Bfun/jeHTtFb6vZpgSRA/vttx+rV6+mW7dudOnShbPOOouTTz6Z4v796XvAvnz7W9+EVYthTdQksmxDOO9p+bvnssNWLQ6nDr78EAb/DgZemt31NWsd2sHLzmfAqHAR+u3bwsOH1n8FR/4Y/uPq+NZteShrCcLMxgEnAUvdff9o3A3AKKA0Knatu0+OmXcw8DegEPiHuydzbqgeffjBB6FHzbXL6VS4lrce+0foUROqEsGqEtZ89hosnQ1Y1KdO06oO1SrflTxiLf4gdKuwYWV45kLvwUlHJA3dd34dHvf55SwYOV5HgWmyeQRxF3ALcE/a+L+4+x8zzWRmhcCtwPFACfCemU1y94+zFWhWuIcjgdROzsrLwrSCJuEGqladQ0+WFV06lJdt2zXzlo2wbu22HZ4VNt26E7bmu+T2+zU0nz4d7l5u0S704Ln7AUlHJDuDgsLQ7Bl0x3uMrCUId3/VzHruwKwDgLnu/gWAmd0PDAMadoLwcti8PkoIUZfJHt19W1AU+qKp6EO/SbP4jbGwKOoBMu1uave05LGpanj912E9qxfDpk2hbL5s6GWbYNF0+HQyvHFTOHf8/Qey13OnNE758v+yA5K4BnG5mZ0LTAV+5O5fp03vBixI+VwCHJJpYWY2GhgN0KNHfI+W7o5layPYshlWloQeKSv28gubhT3ZigerFDat20Zoljl5AJRtxL+eD+sWw/1nwSk3b/04xsaibCMsnA7zXod5r4VnCJStD9P2+264M1pdW4jUm1wniNuB/wd49P4n4MK0MnG/pBnv5nP3scBYCDfKpU9v3rw5y5cvp2PHjvWfJDasCk+vKt8SnhxV8USuHPcD74VNWU57mjdrGh7ofvuhMOw22DvHT9Oqb2UbQ7cV897YNiHstj/0Py90i9HjsMaZEEUSltME4e6Vd4OZ2R3AkzHFSoA9Uj53Bxbt6Dq7d+9OSUkJpaWl2y9cU+5V/dkXFkHLTrByDbBjd0DXh+bNm9P9gCOh64uhbff400IHZcf/qm7dVudSZUJIPUKI7g/Z7QDof35ICHsetlM+vlFkZ5PTBGFmXdx9cfTxu8CsmGLvAXubWS9gITAS+P6OrrOoqIhevXrt6OzbWvF/4WJoybuhDfzgGxvWIwV3PwBGvwzP3xCebfDvV+G0O7J70XZNaXia1pbNoWXWlk1huGxj1XDl+E3bjivbCEs+gpL3ooRgoZfM4gthz8OVEEQSks1mrhOAo4FOZlYC/BI42sz6Ek4ZzQMuicp2JTRnHeruZWZ2OfAsoZnrOHf/KFtx1srHk8IDZtxDlwr7n5Z0RPGKWsCQ34XO5h77Yeik7thfwKGX199doWUbQ8uh9++Dz1/Y/mMlY1n0GMwi6NAzJISeR0CPQ5UQRBqARt9ZX73YvAGmXBf6ku/aLySHDvV4VJJNa5fBpCvCXcW9joJTx4R+iXbUlx+GpFBxY1GbrtD3TOjWv+oejcKm0UX11OFm244vKFQLEpGEVddZn+6k3p7Sz+DhC2DJrPCQkGOv37nusmzVKdwANP0eeOYauP0wOPmvodVPTa37Cj58GN6/F778IPy4f/tE6Hs27HVMo32alki+U4LIxD08SWzyj8Mpm+8/BN86IemodoxZVYufRy4O/eF/NiWchsrUgV35FvjipXC08MlT4VrB7n1C/zUHnK5TQCJ5QAkizsbV8OR/w4cPQs8jQ8ddmfqN35l03AsumgKv/B5e+yPMfyN8tx4pt5ks/zwkxpkTYNVCaNE+XBvoe1boBVNE8oYSRLpFM8Ippa/nwTHXwZE/alynUAqL4Njr4JuD4NFR8M/BoYOyDr3C0cL8N0I/T3sNgu/8BnoPCReSRSTvKEFUcA/NQqf8IvTpf/5ToXllY9VjIFz6Bjz9U3j192Fch71g0PXhWQdtuyYbn4gkTgkCwkXYx34Inz0NvYeGLhvy4Rx787bw3TFw0DnhKGmPQ9SqSEQqKUGs+wrGHAFrS8PzAw65JP9+JHsennQEItIAKUG07AD9Lwg3lXXtm3Q0IiINhhIEwH/8JOkIREQaHD2WTEREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJlbUEYWbjzGypmc1KGfcHM/vEzD4ws4lm1i7DvPPM7EMzm2FmU7MVo4iIZJbNI4i7gMFp454D9nf3PsBnwM+qmf8Yd+/r7sVZik9ERKqRtQTh7q8CX6WNm+LuZdHHt4Hu2Vq/iIjUTZLXIC4Ens4wzYEpZjbNzEZXtxAzG21mU81samlpab0HKSKSrxJJEGZ2HVAGjM9Q5HB37wcMAS4zs6MyLcvdx7p7sbsXd+7cOQvRiojkp5wnCDM7DzgJOMvdPa6Muy+K3pcCE4EBuYtQREQgxwnCzAYDVwOnuPu6DGVamVmbimHgBGBWXFkREcmebDZznQC8BfQ2sxIzuwi4BWgDPBc1YR0Tle1qZpOjWXcDXjezmcC7wFPu/ky24hQRkXhZeya1u58ZM/rODGUXAUOj4S+AA7MVl4iI1IzupBYRkVhKECIiEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYmUtQZjZODNbamazUsZ1MLPnzGxO9N4+w7yDzexTM5trZtdkK0YREcksm0cQdwGD08ZdA7zg7nsDL0Sft2JmhcCtwBBgX+BMM9s3i3GKiEiMrCUId38V+Cpt9DDg7mj4buDUmFkHAHPd/Qt33wTcH80nIiI5lOtrELu5+2KA6H3XmDLdgAUpn0uicbHMbLSZTTWzqaWlpfUarIhIPmuIF6ktZpxnKuzuY9292N2LO3funMWwRETyS64TxBIz6wIQvS+NKVMC7JHyuTuwKAexiYhIilwniEnAedHwecDjMWXeA/Y2s15m1hQYGc0nIiI5lM1mrhOAt4DeZlZiZhcBNwLHm9kc4PjoM2bW1cwmA7h7GXA58CwwG3jQ3T/KVpwiIhKvSbYW7O5nZpg0KKbsImBoyufJwOQshSYiIjXQEC9Si4hIA6AEISIisZQgREQklhKEiIjEqlGCMLMrzaytBXea2XQzOyHbwYmISHJqegRxobuvAk4AOgMXEDVRFRGRxqmmCaKi+4uhwD/dfSbxXWKIiEgjUdMEMc3MphASxLNm1gYoz15YIiKStJreKHcR0Bf4wt3XmVkHwmkmERFppGp6BHEo8Km7rzCzs4GfAyuzF5aIiCStpgnidmCdmR0I/BSYD9yTtahERCRxNU0QZe7uhCe7/c3d/wa0yV5YIiKStJpeg1htZj8DzgGOjJ4bXZS9sEREJGk1PYI4A9hIuB/iS8IjQP+QtahERCRxNUoQUVIYD+xiZicBG9xd1yBERBqxmna1MQJ4FzgdGAG8Y2bDsxmYiIgkq6bXIK4DDnb3pQBm1hl4Hng4W4GJiEiyanoNoqAiOUSW12JeERHZCdX0COIZM3sWmBB9PgM9ElREpFGrUYJw95+Y2WnA4YRO+sa6+8SsRiYiIomq6REE7v4I8EhdV2hmvYEHUkZ9A7je3f+aUuZo4HHg39GoR939f+q6bhERqblqE4SZrQY8bhLg7t62tit0908JHf8R3XC3EIg7GnnN3U+q7fJFRKR+VJsg3D3b3WkMAj539/lZXo+IiNRS0i2RRlJ14TvdoWY208yeNrP9Mi3AzEab2VQzm1paWpqdKEVE8lBiCcLMmgKnAA/FTJ4O7OnuBwI3A49lWo67j3X3Yncv7ty5c1ZiFRHJR0keQQwBprv7kvQJ7r7K3ddEw5OBIjPrlOsARUTyWZIJ4kwynF4ys93NzKLhAYQ4l+cwNhGRvFfjZq71ycxaAscDl6SMuxTA3ccAw4EfmFkZsB4YGT2PQkREciSRBOHu64COaePGpAzfAtyS67hERKRK0q2YRESkgVKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiJZIgzGyemX1oZjPMbGrMdDOzm8xsrpl9YGb9kohTRCSfNUlw3ce4+7IM04YAe0evQ4Dbo3cREcmRhnqKaRhwjwdvA+3MrEvSQYmI5JOkEoQDU8xsmpmNjpneDViQ8rkkGrcNMxttZlPNbGppaWkWQhURyU9JJYjD3b0f4VTSZWZ2VNp0i5nH4xbk7mPdvdjdizt37lzfcYqI5K1EEoS7L4relwITgQFpRUqAPVI+dwcW5SY6ERGBBBKEmbUyszYVw8AJwKy0YpOAc6PWTAOBle6+OMehiojktSRaMe0GTDSzivX/y92fMbNLAdx9DDAZGArMBdYBFyQQp4hIXst5gnD3L4ADY8aPSRl24LJcxiUiIltrqM1cRUQkYUoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERi5TxBmNkeZvaSmc02s4/M7MqYMkeb2UozmxG9rs91nCIi+a5JAussA37k7tPNrA0wzcyec/eP08q95u4nJRCfiIiQwBGEuy929+nR8GpgNtAt13GIiEj1Er0GYWY9gYOAd2ImH2pmM83saTPbL7eRiYhIEqeYADCz1sAjwFXuvipt8nRgT3dfY2ZDgceAvTMsZzQwGqBHjx7ZC1hEJM8kcgRhZkWE5DDe3R9Nn+7uq9x9TTQ8GSgys05xy3L3se5e7O7FnTt3zmrcIiL5JIlWTAbcCcx29z9nKLN7VA4zG0CIc3nuohQRkSROMR0OnAN8aGYzonHXAj0A3H0MMBz4gZmVAeuBke7uCcQqIpK3cp4g3P11wLZT5hbgltxEJCIicXQntYiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiJZIgzGywmX1qZnPN7JqY6WZmN0XTPzCzfknEKSKSz5rkeoVmVgjcChwPlADvmdkkd/84pdgQYO/odQhwe/SeFRPe/T/K3Skwo9AMMygsMArShsMr5XNB9NkMjMoyZlBgYGYYpIwL74ZRUBB9DnWSFpFv/cmrmxqmO447lHt4rxhX7uAe3qn8nFrOcaiMozD6TqnDld+9gKh+tq4Hs7jvkBpfesSZWWWdRJ+pqLOKAlXjSIm7puuNi2Tb+q2+wuOWURFHan1ZAZXbTOr2U5M6q6m471jX5VZsL+XuldtJeco4L6+aVh5tZ3jm75/+P1CXOqj4vhVf29PHV36umO4p86Yva9vlV25XtvW2l7pd1sffbWeR8wQBDADmuvsXAGZ2PzAMSE0Qw4B7PPzV3zazdmbWxd0XZyOgXz3xERs2l2dj0SIZpf5gGlb5Yxb345f+w1fb9VQOV46zrT5DVRLIpdQfYMj8Q98QxSaPsAez1ef0nRmLPlR+ThkuiCZW7mRStcMWklPMOKBjq2Y8eOmh9f4dk0gQ3YAFKZ9L2PboIK5MN2CbBGFmo4HRAD169NihgF776bG4O1sq9pLKw97RlvKqPfAt7pSn7DlVTKsYjtuLr9xLJ8zrVIyrmF61vPS9kvR9lPSdFksrUbFBkbLxFFRsUHFHMmy9gQFsSYm74julf9fytDqqGN6umux0pdRh9LGy/qgcpvJXo3J6TP2l1kuNQqlj/W91tJZSX+5E21VUn1tNC8NbUn4Fq37At17P1kdRFlsWtk0uqb+w29u7LiyoOjqsTFwxRz6Vnwu2PgIOf49tv395lOCqqwPHt/quW/0NMnzfTOWrjgK2/UtnmreiDrba/ir/f7eeVrENxo2v3F69qj6qlh2/DqCyDip+F9jmtyRlPZX1WRVjm+bZ+SlPIkHE/X+m/8LUpEwY6T4WGAtQXFy8Q/sbnds025HZREQatSQuUpcAe6R87g4s2oEyIiKSRUkkiPeAvc2sl5k1BUYCk9LKTALOjVozDQRWZuv6g4iIxMv5KSZ3LzOzy4FngUJgnLt/ZGaXRtPHAJOBocBcYB1wQa7jFBHJd0lcg8DdJxOSQOq4MSnDDlyW67hERKSK7qQWEZFYShAiIhJLCUJERGIpQYiISCyrTT85DZ2ZlQLzd3D2TsCyegynvim+ulF8daP46qYhx7enu3eOm9CoEkRdmNlUdy9OOo5MFF/dKL66UXx109Djy0SnmEREJJYShIiIxFKCqDI26QC2Q/HVjeKrG8VXNw09vli6BiEiIrF0BCEiIrGUIEREJFZeJQgzG2xmn5rZXDO7Jma6mdlN0fQPzKxfjuPbw8xeMrPZZvaRmV0ZU+ZoM1tpZjOi1/U5jnGemX0YrXtqzPTE6tDMeqfUywwzW2VmV6WVyWn9mdk4M1tqZrNSxnUws+fMbE703j7DvNVur1mM7w9m9kn095toZu0yzFvttpDF+G4ws4Upf8OhGeZNqv4eSIltnpnNyDBv1uuvzrzyEZiN+0XoWvxz4BtAU2AmsG9amaHA04Qn2g0E3slxjF2AftFwG+CzmBiPBp5MsB7nAZ2qmZ5oHab9vb8k3ASUWP0BRwH9gFkp434PXBMNXwP8LkP81W6vWYzvBKBJNPy7uPhqsi1kMb4bgB/X4O+fSP2lTf8TcH1S9VfXVz4dQQwA5rr7F+6+CbgfGJZWZhhwjwdvA+3MrEuuAnT3xe4+PRpeDcwmPIt7Z5JoHaYYBHzu7jt6Z329cPdXga/SRg8D7o6G7wZOjZm1JttrVuJz9ynuXhZ9fJvwRMdEZKi/mkis/ipYeCj2CGBCfa83V/IpQXQDFqR8LmHbH9+alMkJM+sJHAS8EzP5UDObaWZPm9l+uY0MB6aY2TQzGx0zvaHU4Ugy/2MmWX8Au3n0hMTofdeYMg2lHi8kHBHG2d62kE2XR6fAxmU4RdcQ6u9IYIm7z8kwPcn6q5F8ShAWMy69jW9NymSdmbUGHgGucvdVaZOnE06bHAjcDDyW4/AOd/d+wBDgMjM7Km164nVo4VG2pwAPxUxOuv5qqiHU43VAGTA+Q5HtbQvZcjuwF9AXWEw4jZMu8foDzqT6o4ek6q/G8ilBlAB7pHzuDizagTJZZWZFhOQw3t0fTZ/u7qvcfU00PBkoMrNOuYrP3RdF70uBiYRD+VSJ1yHhH266uy9Jn5B0/UWWVJx2i96XxpRJtB7N7DzgJOAsj06Yp6vBtpAV7r7E3be4ezlwR4b1Jl1/TYDvAQ9kKpNU/dVGPiWI94C9zaxXtIc5EpiUVmYScG7UEmcgsLLiVEAuROcs7wRmu/ufM5TZPSqHmQ0g/A2X5yi+VmbWpmKYcDFzVlqxROswknHPLcn6SzEJOC8aPg94PKZMTbbXrDCzwcDVwCnuvi5DmZpsC9mKL/Wa1nczrDex+oscB3zi7iVxE5Osv1pJ+ip5Ll+EFjafEVo3XBeNuxS4NBo24NZo+odAcY7jO4JwGPwBMCN6DU2L8XLgI0KrjLeBw3IY3zei9c6MYmiIddiS8IO/S8q4xOqPkKgWA5sJe7UXAR2BF4A50XuHqGxXYHJ122uO4ptLOH9fsQ2OSY8v07aQo/jujbatDwg/+l0aUv1F4++q2OZSyua8/ur6UlcbIiISK59OMYmISC0oQYiISCwlCBERiaUEISIisZQgREQklhKESANgoZfZJ5OOQySVEoSIiMRSghCpBTM728zejfrw/7uZFZrZGjP7k5lNN7MXzKxzVLavmb2d8lyF9tH4b5rZ81GHgdPNbK9o8a3N7GELz2IYX3HHt0hSlCBEasjM9gHOIHSy1hfYApwFtCL0/dQPeAX4ZTTLPcDV7t6HcOdvxfjxwK0eOgw8jHAnLoTee68C9iXcaXt4lr+SSLWaJB2AyE5kENAfeC/auW9B6GivnKpO2e4DHjWzXYB27v5KNP5u4KGo/51u7j4RwN03AETLe9ejvnuip5D1BF7P+rcSyUAJQqTmDLjb3X+21UizX6SVq67/mupOG21MGd6C/j8lYTrFJFJzLwDDzWxXqHy29J6E/6PhUZnvA6+7+0rgazM7Mhp/DvCKh+d7lJjZqdEymplZy1x+CZGa0h6KSA25+8dm9nPCU8AKCD14XgasBfYzs2nASsJ1CghdeY+JEsAXwAXR+HOAv5vZ/0TLOD2HX0OkxtSbq0gdmdkad2+ddBwi9U2nmEREJJaOIEREJJaOIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERi/X9r2A1ESoydTgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history3.history['loss'])\n",
    "plt.plot(history3.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6f170c3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7178, 128, 128, 3)\n",
      "(7178, 7)\n",
      "(7178, 7)\n",
      "(7178, 7)\n",
      "50246\n",
      "[[False  True False ...  True  True  True]\n",
      " [ True  True  True ...  True  True  True]\n",
      " [False  True  True ...  True False  True]\n",
      " ...\n",
      " [ True  True False ...  True  True False]\n",
      " [False  True  True ...  True  True False]\n",
      " [ True  True False ...  True  True False]]\n",
      "Sum is:  42982\n",
      "y_train.shape[0] =  7178\n",
      "Accuracy on the testing set is:  49.400947339091665\n"
     ]
    }
   ],
   "source": [
    "# Accuracy of custom built model (3):\n",
    "# X_test /= 255.0\n",
    "\n",
    "print(y_test.shape)\n",
    "predictions = model3.predict(X_test)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum = np.sum(classes == y_test)\n",
    "\n",
    "accuracy = (sum - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c050b2eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_13\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_62 (Conv2D)           (None, 128, 128, 32)      896       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_61 (MaxPooling (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_73 (Batc (None, 64, 64, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv2d_63 (Conv2D)           (None, 64, 64, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_62 (MaxPooling (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_74 (Batc (None, 32, 32, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_64 (Conv2D)           (None, 32, 32, 128)       73856     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_63 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_75 (Batc (None, 16, 16, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv2d_65 (Conv2D)           (None, 16, 16, 256)       295168    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_64 (MaxPooling (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_66 (Conv2D)           (None, 6, 6, 256)         590080    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_65 (MaxPooling (None, 3, 3, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_76 (Batc (None, 3, 3, 256)         1024      \n",
      "_________________________________________________________________\n",
      "flatten_12 (Flatten)         (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_77 (Batc (None, 2304)              9216      \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 7)                 16135     \n",
      "=================================================================\n",
      "Total params: 1,005,767\n",
      "Trainable params: 1,000,199\n",
      "Non-trainable params: 5,568\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# MODEL4 (Custom):\n",
    "\n",
    "model4 = Sequential()\n",
    "\n",
    "model4.add(Conv2D(32, kernel_size = (3,3), activation = 'relu', padding = 'same', input_shape = (img_size, img_size, 3)))\n",
    "model4.add(MaxPool2D(pool_size=(2,2)))\n",
    "model4.add(BatchNormalization())\n",
    "\n",
    "model4.add(Conv2D(64, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model4.add(MaxPool2D(pool_size=(2,2)))\n",
    "model4.add(BatchNormalization())\n",
    "\n",
    "model4.add(Conv2D(128, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model4.add(MaxPool2D(pool_size=(2,2)))\n",
    "model4.add(BatchNormalization())\n",
    "\n",
    "model4.add(Conv2D(256, kernel_size = (3,3), activation = 'relu', padding = 'same'))\n",
    "model4.add(MaxPool2D(pool_size=(2,2)))\n",
    "\n",
    "model4.add(Conv2D(256, kernel_size = (3,3), activation = 'relu', padding = 'valid'))\n",
    "model4.add(MaxPool2D(pool_size=(2,2)))\n",
    "model4.add(BatchNormalization())\n",
    "\n",
    "model4.add(Flatten())\n",
    "model4.add(BatchNormalization())\n",
    "\n",
    "model4.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model4.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e31e600",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same model trained with DATA AUGMENTATION:\n",
    "# ==> RAM can't handle data augmentation. So training fails\n",
    "\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "# create data generator\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=10,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    shear_range=0.1,\n",
    "    zoom_range=0.2,\n",
    "    horizontal_flip=True,\n",
    ")\n",
    "\n",
    "# prepare iterator\n",
    "it_train = datagen.flow(X_train, y_train, batch_size=64)\n",
    "\n",
    "\n",
    "# fit model:\n",
    "steps = int(X_train.shape[0] / 64)\n",
    "model4.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "history4 = model4.fit(it_train, steps_per_epoch=steps, epochs=3, validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3ab24264",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenetv2_1.00_128 (Model) (None, 4, 4, 1280)        2257984   \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 20480)             0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 20480)             81920     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 7)                 143367    \n",
      "=================================================================\n",
      "Total params: 2,483,271\n",
      "Trainable params: 2,408,199\n",
      "Non-trainable params: 75,072\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model5 using the mobile_net as the base model. The model was trained on google colab for 1hr. Data augmentation was used.\n",
    "\n",
    "model5 = Sequential()\n",
    "model5.add(mobilenet_model)\n",
    "\n",
    "model5.add(Flatten())\n",
    "model5.add(BatchNormalization())\n",
    "\n",
    "model5.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model5.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "877c2c4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Weights downloaded from colab:\n",
    "model5.load_weights(os.path.join(Project_DIR, \"model_mobilenet\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a2403c5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7178, 7)\n",
      "(7178, 7)\n",
      "Accuracy on the testing set is:  39.885762050710504\n"
     ]
    }
   ],
   "source": [
    "# Predictions of the model:\n",
    "\n",
    "print(y_test.shape)\n",
    "predictions = model5.predict(X_test)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum = np.sum(classes == y_test)\n",
    "\n",
    "accuracy = (sum - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "54fe19aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenetv2_1.00_224 (Model) (None, 7, 7, 1280)        2257984   \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 62720)             0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 62720)             250880    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 7)                 439047    \n",
      "=================================================================\n",
      "Total params: 2,947,911\n",
      "Trainable params: 2,788,359\n",
      "Non-trainable params: 159,552\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# model6 using the mobile_net as the base model, trained on Kaggle. Input = (224, 224)\n",
    "img_size = 224\n",
    "mobilenet_model = tf.keras.applications.MobileNetV2(input_shape=(img_size, img_size, 3), include_top=False, weights='imagenet')\n",
    "\n",
    "model6 = Sequential()\n",
    "model6.add(mobilenet_model)\n",
    "\n",
    "model6.add(Flatten())\n",
    "model6.add(BatchNormalization())\n",
    "\n",
    "model6.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model6.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6b256ce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model trained on Colab\n",
    "# model6 = Sequential()\n",
    "model6.load_weights(os.path.join(Project_DIR, \"model_mobilenet\", \"model_mobilenet_input_224_50_epochs.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a6985cfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "resnet50 (Model)             (None, 7, 7, 2048)        23587712  \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 100352)            0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 100352)            401408    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 7)                 702471    \n",
      "=================================================================\n",
      "Total params: 24,691,591\n",
      "Trainable params: 20,356,103\n",
      "Non-trainable params: 4,335,488\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model7 using the resnet_model as the base model, trained on colab. Input = (224, 224)\n",
    "img_size = 224\n",
    "resnet_model = ResNet50(input_shape=(img_size, img_size, 3), include_top=False, weights='imagenet')\n",
    "\n",
    "for layer in resnet_model.layers:\n",
    "   layer.trainable = False\n",
    "for layer in resnet_model.layers[-75:]:\n",
    "   layer.trainable = True\n",
    "\n",
    "model7 = Sequential()\n",
    "model7.add(resnet_model)\n",
    "\n",
    "model7.add(Flatten())\n",
    "model7.add(BatchNormalization())\n",
    "\n",
    "model7.add(Dense(classes_num, activation='softmax'))\n",
    "\n",
    "model7.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "57671923",
   "metadata": {},
   "outputs": [],
   "source": [
    "# **REMEMBER**: Always save and load model weights in .h5 format when training on kaggle or colab\n",
    "\n",
    "# Model7 trained on Kaggle:\n",
    "\n",
    "model7.load_weights(os.path.join(Project_DIR, 'model_resnet_FineTuned_Large', \"model_resnet_FineTuned_100ep.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fef6d1b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\anaconda3\\envs\\tensorflow_env\\lib\\site-packages\\ipykernel_launcher.py:42: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3705, 224, 224, 3)\n",
      "(3705, 7)\n"
     ]
    }
   ],
   "source": [
    "# Testing models 6 and 7:\n",
    "\n",
    "img_size = 224\n",
    "\n",
    "def label_img(category):\n",
    "    arr = np.zeros((classes_num,), dtype = int)\n",
    "    arr[classes.index(category), ] = 1\n",
    "    return arr\n",
    "\n",
    "test = []\n",
    "# train = []\n",
    "\n",
    "def create_test_set(img_size):\n",
    "    for category in classes:\n",
    "        imgs = 0\n",
    "        category_path = os.path.join(Test_DIR, category)\n",
    "        label = label_img(category)\n",
    "        for img in os.listdir(category_path):\n",
    "            imgs = imgs + 1\n",
    "            if imgs == 600:\n",
    "                break\n",
    "            img_array = cv2.resize(cv2.imread(os.path.join(category_path, img)), (img_size, img_size))\n",
    "            test.append([np.array(img_array), np.array(label)])\n",
    "\n",
    "# def create_train_set(img_size):\n",
    "#     for category in classes:\n",
    "#         imgs = 0\n",
    "#         category_path = os.path.join(Train_DIR, category)\n",
    "#         label = label_img(category)\n",
    "#         for img in os.listdir(category_path):\n",
    "#             imgs = imgs + 1\n",
    "#             if imgs == 600:\n",
    "#                 break\n",
    "#             img_array = cv2.resize(cv2.imread(os.path.join(category_path, img)), (img_size, img_size))\n",
    "#             train.append([np.array(img_array), np.array(label)])\n",
    "\n",
    "        \n",
    "create_test_set(img_size)\n",
    "# create_train_set(img_size)\n",
    "\n",
    "# train = np.array(train)\n",
    "test = np.array(test)\n",
    "# shuffle(train)\n",
    "shuffle(test)\n",
    "\n",
    "X_test = np.array([i[0] for i in test]).reshape(-1, img_size, img_size, 3)\n",
    "y_test = np.array([i[1] for i in test])\n",
    "\n",
    "# X_train = np.array([i[0] for i in train]).reshape(-1, img_size, img_size, 3)\n",
    "# y_train = np.array([i[1] for i in train])\n",
    "\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "\n",
    "# print(X_train.shape)\n",
    "# print(y_train.shape)\n",
    "\n",
    "del test\n",
    "# del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e05df3c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3705, 7)\n",
      "(3705, 7)\n",
      "Accuracy on the testing set is:  59.32523616734143\n"
     ]
    }
   ],
   "source": [
    "# Model6: Optimized with Adam and learning rate = 3e-4\n",
    "# (For the entire test set)\n",
    "# Training Accuracy = 60.06% after 50 epochs\n",
    "# Training Accuracy = 60.67%  after 70 epochs (near peak performance, after which validation loss increases)\n",
    "\n",
    "print(y_test.shape)\n",
    "predictions = model6.predict(X_test)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum_ = np.sum(classes == y_test)\n",
    "accuracy = (sum_ - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "999849b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4030, 7)\n",
      "(4030, 7)\n",
      "Accuracy on the training set is:  78.85856079404466\n"
     ]
    }
   ],
   "source": [
    "# As expected, predictions on a subset of the training set are very accurate:\n",
    "# But the accuracy is still less than 90% which was observed while training the model on Kaggle notebook\n",
    "# This suggests that accuracy depends on the subset of training set we choose. This is not surprising as the dataset contains\n",
    "# far more images for happy or neutral as compared to disgust. This is also the reason why all\n",
    "# the models built so far are unable to recognize disgust\n",
    "\n",
    "print(y_train.shape)\n",
    "predictions = model6.predict(X_train)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum_ = np.sum(classes == y_train)\n",
    "accuracy = (sum_ - (y_train.shape[0] * (classes_num - 2))) / (2 * y_train.shape[0])\n",
    "\n",
    "print('Accuracy on the training set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3a324e4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3705, 7)\n",
      "(3705, 7)\n",
      "Accuracy on the testing set is:  58.65047233468286\n"
     ]
    }
   ],
   "source": [
    "# Model7: Optimized with Adam and learning rate = 3e-4\n",
    "\n",
    "# Note that the number of examples from each category has been limited to 600 in the testing set.\n",
    "# This is mainly done to reduce the computational cost of prediction. \n",
    "\n",
    "# Model 7 starts overfitting early and validation accuracy seems to be constant at around 67% after 10 epochs\n",
    "# Training Accuracy =  59.11% after 25 epochs\n",
    "# Training Accuracy = 58.65%  after 100 epochs (near peak performance, after which validation loss increases)\n",
    "\n",
    "print(y_test.shape)\n",
    "predictions = model7.predict(X_test)\n",
    "print(predictions.shape)\n",
    "classes = np.array(tf.one_hot(tf.argmax(predictions, axis = 1), depth = classes_num))\n",
    "classes = classes.astype(int)\n",
    "sum = np.sum(classes == y_test)\n",
    "accuracy = (sum - (y_test.shape[0] * (classes_num - 2))) / (2 * y_test.shape[0])\n",
    "\n",
    "print('Accuracy on the testing set is: ', accuracy * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46768246",
   "metadata": {},
   "source": [
    "# Live Webcam Testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e7cbe9ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code inspired from: https://medium.com/analytics-vidhya/realtime-face-emotion-recognition-using-transfer-learning-in-tensorflow-3add4f4f3ff3\n",
    "\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "img_size = 224\n",
    "classes = ['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise']\n",
    "\n",
    "path = os.path.join(Project_DIR, \"haarcascade_frontalface_default.xml\")\n",
    "font_scale = 1.5\n",
    "font = cv2.FONT_HERSHEY_PLAIN\n",
    "\n",
    "rectangle_bgr = (255, 255, 255)\n",
    "img = np.zeros((500, 500))\n",
    "text = \"Hello\"\n",
    "(text_width, text_height) = cv2.getTextSize(text, font, fontScale = font_scale, thickness = 1)[0]\n",
    "\n",
    "text_offset_x = 10\n",
    "text_offset_y = img.shape[0] - 25\n",
    "\n",
    "# 2 pixels for padding\n",
    "box_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y - text_height - 2))\n",
    "cv2.rectangle(img, box_coords[0], box_coords[1], rectangle_bgr, cv2.FILLED)\n",
    "cv2.putText(img, text, (text_offset_x, text_offset_y), font, fontScale = font_scale, color = (0, 0, 0), thickness = 1)\n",
    "\n",
    "face_roi = X_test[1000]\n",
    "x = 1\n",
    "y = 1\n",
    "w = 1\n",
    "h = 1\n",
    "cap = cv2.VideoCapture(1)\n",
    "if not cap.isOpened():\n",
    "    cap = cv2.VideoCapture(0)\n",
    "if not cap.isOpened():\n",
    "    raise IOError(\"Cannot open webcam!\")\n",
    "\n",
    "while True:\n",
    "    ret, frame = cap.read()\n",
    "    faceCascade = cv2.CascadeClassifier(path)\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "    faces = faceCascade.detectMultiScale(gray, 1.1, 4)  # 1.1 is the scaleFactor (google), 4 : min neighbours\n",
    "    for x, y, w, h in faces:\n",
    "        roi_gray = gray[y : y + h, x : x + w]  # roi = region of interest\n",
    "        roi_color = frame[y : y + h, x : x + w]\n",
    "        cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2)\n",
    "        facess = faceCascade.detectMultiScale(roi_gray)\n",
    "        if len(facess) != 0:\n",
    "            for (ex, ey, ew, eh) in facess:\n",
    "                face_roi = roi_color[ey: ey + eh, ex : ex + ew]  # crop the face to get the roi\n",
    "                \n",
    "    # RESIZE THE IMAGE:\n",
    "    final_image = cv2.resize(face_roi, (img_size, img_size))\n",
    "    final_image = np.expand_dims(final_image, axis = 0)\n",
    "    \n",
    "    font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "    \n",
    "    # SET THE MODEL to be used for prediction:\n",
    "    Predictions = model6.predict(final_image)\n",
    "    class_num = np.argmax(Predictions)   # **Provides the index of the max argument\n",
    "    \n",
    "    status = classes[class_num]\n",
    "    x1, y1, w1, h1 = 0, 0, 175, 75\n",
    "    cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0, 0, 0), -1)\n",
    "    cv2.putText(frame, status, (x1 + int(w1 / 10), y1 + int(h1 / 2)), font, 0.7, (0, 0, 255), 2)\n",
    "    cv2.putText(frame, status, (100, 150), font, 3, (0, 0, 255), 2, cv2.LINE_4)\n",
    "    cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255))\n",
    "    \n",
    "    cv2.imshow(\"Live Facial Expression Recognition\", frame)\n",
    "    \n",
    "    # The following line is to enable closing of webcam using 'q' key\n",
    "    if cv2.waitKey(2) & 0xFF == ord('q'):\n",
    "        break\n",
    "        \n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a0bd839",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Progress:\n",
    "\n",
    "# 1) model3 can easily detect these emotions: happy, neutral, angry and fear. It is also the model with highest testing accuracy\n",
    "#    that could achieve 50 % within just 1.5 hours of training. With that being said, it highly overfit the dataset and \n",
    "#    the validation loss only kept growing with epochs. It is certainly useless to train the model any furhter\n",
    "\n",
    "# 2) Although model2 provides low accuracy currently, it could attain high accuracy if it were left to train for a few more hours\n",
    "\n",
    "# 3) Since model2 demands more training time, I modified it and fine tuned mobileNet. Model1 is replaced with this modeified model.\n",
    "#    Update1: Model1 seems to be more promising than model2 and I will continue to train it for a few hours\n",
    "#    Update2: Even Model1 starts overfitting early and the testing accuracy keeps fluctuating.\n",
    "\n",
    "# 4) model5 trained on colab can easily detect these emotions: happy, neutral, angry. This is still worse than the custom built\n",
    "#    model3(which also had very few training parameters). Now I will try to train model5 on (224, 224) images as it seems\n",
    "#    to be a better model than model3. This model will be model6\n",
    "\n",
    "# 5) After tuning the hyperparameters carefully and training the models 6 and 7 for several hours,\n",
    "#    I have managed to achieve a high accuracy on both of the models. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39620ddc",
   "metadata": {},
   "source": [
    "# Final Results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6869e7e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# (Only results of models 6 and 7 are mentioned as those are the most accurate and meticulously trained models)\n",
    "\n",
    "###############################################################################################################################\n",
    "#                                                                                                                             #\n",
    "#  Model6:                                                                                                                    #\n",
    "#  Based on MobileNetV2                                                                                                       #\n",
    "#  Peak Testing Accuracy = 60.67%                                                                                             #\n",
    "#                                                                                                                             #\n",
    "#  Can detect:                                                                                                                #\n",
    "#                                                                                                                             #\n",
    "#  Always (almost): Happy, Neutral, Angry (prediction is slightly unstable), Sad, Fear                                        #\n",
    "#  Often: Disgust (it shows if you make some specific faces)                                                                  #\n",
    "#  Rarely: Surprise (model confuses it for fear)                                                                              #\n",
    "#                                                                                                                             #\n",
    "#                                                                                                                             #\n",
    "#  Model7:                                                                                                                    #\n",
    "#  Based on Resnet50                                                                                                          #\n",
    "#  Peak Training Accuracy = 59.11%                                                                                            #\n",
    "#  (Couldn't test it on webcam)                                                                                               #\n",
    "#                                                                                                                             #\n",
    "#  Remarks:                                                                                                                   #\n",
    "#  1) Overall model6 has the best testing accuracy and it also performs good on the live webcam test.                         #\n",
    "#  2) The webcam becomes laggy with model7 as resnet50 has a lot of parameters. Because of this, model is unable to           #\n",
    "#     predict on live webcam. Nonetheless, it should perform at most as good as model6, given their testing accuracies.       #\n",
    "#  3) Test the live model with a homogeneous background and good illumination for better results.                             #\n",
    "#  4) (unrelated) Kaggle GPUs are faster than Colab's GPUs and runtimes are more stable. But memory managment is complex      #\n",
    "#     on Kaggle notebooks and you must cautiously use RAM.                                                                    #\n",
    "#  5) (unrelated) Save and download model weights in .h5 format after training your models online.                            #\n",
    "#                                                                                                                             #\n",
    "###############################################################################################################################"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}