notebooks/3_model_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import h5py\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import onnx\n",
    "\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_full_model = \"../models/full_model.pt\"\n",
    "path_converted_model = \"../models/full_model.onnx\"\n",
    "\n",
    "path_dataset = \"../datasets/dataset.h5py\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "h5f = h5py.File(path_dataset, 'r')\n",
    "\n",
    "X_train = np.array(h5f['X_train'])\n",
    "X_test = np.array(h5f['X_test'])\n",
    "y_train = np.array(h5f['y_train'])\n",
    "y_test = np.array(h5f['y_test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.astype(np.float32)\n",
    "X_test = X_test.astype(np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model loading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "\n",
    "        # Features detector\n",
    "        self.features1 = nn.Sequential(\n",
    "            \n",
    "            # Hidden layer 1\n",
    "            nn.Conv2d(3, 16, kernel_size=3), nn.ReLU(),\n",
    "            nn.BatchNorm2d(16),\n",
    "\n",
    "            nn.MaxPool2d((2, 2)),\n",
    "            nn.Dropout(0.35)\n",
    "        )\n",
    "\n",
    "        # Features detector\n",
    "        self.features2 = nn.Sequential(\n",
    "            \n",
    "            # Hidden layer 3\n",
    "            nn.Conv2d(16, 32, kernel_size=3), nn.ReLU(),\n",
    "            nn.BatchNorm2d(32),\n",
    "\n",
    "            nn.MaxPool2d((2, 2)),\n",
    "            nn.Dropout(0.35)\n",
    "        )\n",
    "\n",
    "        # Features detector\n",
    "        self.features3 = nn.Sequential(\n",
    "            \n",
    "            # Hidden layer 3\n",
    "            nn.Conv2d(32, 64, kernel_size=3), nn.ReLU(),\n",
    "            nn.BatchNorm2d(64),\n",
    "\n",
    "            nn.MaxPool2d((2, 2)),\n",
    "            nn.Dropout(0.35)\n",
    "        )\n",
    "\n",
    "        # Classifier\n",
    "        self.classifier = nn.Sequential(\n",
    "            \n",
    "            nn.Linear(64*13*13, 512), nn.ReLU(),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.Dropout(0.50),\n",
    "\n",
    "            # Output layer\n",
    "            nn.Linear(512, 108)\n",
    "        )\n",
    "\n",
    "    def forward(self, X):\n",
    "\n",
    "        # Features\n",
    "        X = self.features1(X)\n",
    "        X = self.features2(X)\n",
    "        X = self.features3(X)\n",
    "\n",
    "        # print(X.shape)\n",
    "\n",
    "        # Classifier\n",
    "        X = X.view(-1, 64*13*13)\n",
    "        #X = X.view(X.size(0), -1)\n",
    "        X = self.classifier(X)\n",
    "\n",
    "        return F.log_softmax(X, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN(\n",
       "  (features1): Sequential(\n",
       "    (0): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "    (4): Dropout(p=0.35, inplace=False)\n",
       "  )\n",
       "  (features2): Sequential(\n",
       "    (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "    (4): Dropout(p=0.35, inplace=False)\n",
       "  )\n",
       "  (features3): Sequential(\n",
       "    (0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "    (4): Dropout(p=0.35, inplace=False)\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=10816, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Dropout(p=0.5, inplace=False)\n",
       "    (4): Linear(in_features=512, out_features=128, bias=True)\n",
       "    (5): ReLU()\n",
       "    (6): Linear(in_features=128, out_features=6, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = torch.load(path_full_model)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specific preprocessing has to be applied for PyTorch model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(755, 3, 120, 120)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([755, 1, 3, 120, 120])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_tensor = torch.tensor(np.array([np.expand_dims(image, 0) for image in X_test]), device=device)\n",
    "X_test_tensor.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "t1 = time.time()\n",
    "\n",
    "for image in X_test_tensor:\n",
    "    results.append(model(image).detach())\n",
    "\n",
    "cost_time = time.time() - t1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inference time: **0.6166** sec, for **755** images.\n",
    "\n",
    "Time per image: **0.008 sec**/image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "755"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost time:  0.6166174411773682\n"
     ]
    }
   ],
   "source": [
    "print(\"cost time: \", cost_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time per image:  0.0008167118426190307\n"
     ]
    }
   ],
   "source": [
    "print(\"time per image: \", cost_time / len(results))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_np = np.array([entry.cpu().numpy() for entry in results]).squeeze()\n",
    "prediction = results_np.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcoAAAHSCAYAAABsGYsqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAp50lEQVR4nO3de3xU5bX/8e+aEJSbgHgjCRoU6q3egdZaLcqpWIsCp/2hvoq1LTVVUaG2Uqz02FOLl58WL7VechBBWyjY1qLC76i1UotX0FKVoAiIkgQEFEQFJSTr90emNCBMQnj27GTvz9vXfjGzZ2bPWsyYxXr2s5+YuwsAAOxYJu4AAABoySiUAADkQKEEACAHCiUAADlQKAEAyIFCCQBADm2ifoP/KRmemutPLl79VNwhICK37n9q3CHkVZ/ajXGHkDdfXvtC3CHk1ZbNVRbVsWvWLgv+875wn4Mji7ep6CgBAMgh8o4SAJASdbVxRxAJOkoAQKtmZpPMbLWZvdZg301m9rqZvWJmD5lZlwaPXWVmS8zsDTMb2NjxKZQAgDC8LvzWNJMlnbHdvickfd7dj5a0WNJVkmRmR0g6V9KR2dfcaWYFuQ5OoQQAtGru/rSk97fb97i7b8nefV5SSfb2YEm/d/dP3f0tSUsk9ct1fM5RAgDCqGtyB5hv35M0PXu7WPWF818qs/t2ikIJAAjCmz5U2mRmViaprMGucncv34XXXy1pi6TfNTcGCiUAoMXKFsUmF8aGzOw7kgZJGuD//p2SVZJ6NHhaSXbfTnGOEgAQRl1d+K2ZzOwMSWMkne3uDVfQeFjSuWa2h5n1lNRb0ou5jkVHCQBo1cxsmqT+kvYxs0pJ16h+lusekp4wM0l63t0vcveFZjZDUoXqh2RHunvOC0AplACAMCI4R9mkt3U/bwe7783x/PGSxjf1+BRKAEAYrMwDAED60FECAMKIaeg1anSUAADkQEcJAAij5a7Ms1solACAIKJYmaclYOgVAIAc6CgBAGEkdOiVjhIAgBzoKAEAYXCOEgCA9KGjBACEkdAl7CiUAIAwGHoFACB96CgBAGFweQgAAOlDRwkACCOh5ygplACAMBh6BQAgfRLTUVrGNGT2tdq4ap0e+86vdNYff6bCjntKkvbstpfWLFiqJ75/a7xBRmDg6f01YcIvVJDJaNJ90/R/b/pN3CFFJi25WsY0bNa1+njVOj363V9pwIQyFX/hMH364SZJ0pNX3KO1Fe/EHOXuK9irvQ6+eaTaH9ZDcmnpFXdo09Jq9b77R9qjZF99WrlGb/7gZtV+8HHcoQaX1O+yO9dRtmifH3GG1i+pVtuO7SRJj3zj2q2P/Uf55Xr7sZfjCi0ymUxGt982XmeceZ4qK1fq+edm65FHH9eiRW/GHVpwacr1mBFnaF2D77IkPTN+mpbOnhdjVOGV/mKE1s/5h94su0lW2EaZdm1VfPk3tGHuK6q+4yEVXTpUxZf+p94Z/0DcoQaVpu9yUiRi6LVD973VY8CxemPqnM88VtixnYq+dKSWP/ZS/gOLWL++x2np0uV66613VFNToxkzZursswbGHVYk0pJrhwP21kGnHauF0+bEHUqkCjq1V6cvHqE1U/8iSfKaLardsFFdB/bTmhlzJElrZsxR1zP6xRhlNBL9Xfa68FsL0GhHaWaHSRosqTi7q0rSw+6+KMrAdsUXfz5cL46fpsIG/wL/l9KBJ6jqmYWq+WhTDJFFq6j4AK2orN56v7Jqpfr1PS7GiKKTllxP/vlwPXvdNLXtsO13+Ytjhqnv6KGqfGahnr1+uuo2b4kpwjD2OHA/bXlvgw655VK1P7JUH7+yTMt/dq8K9+mimtXrJEk1q9epcJ8u8QYagUR/l9M4mcfMfiLp95JM0ovZzSRNM7Ox0YfXuAMHHKtP1m7Q2leX7/DxQ4acqKUzn8tvUEAzlA44Vpve26A1232Xn7thhn7X/0rNGPRf2rNzR51w8aB4AgzICgrU4aiD9e79j+nV03+s2o2fqOjS//zsE93zHxywncY6yhGSjnT3moY7zWyCpIWSbtjRi8ysTFKZJA3v0k+ndOgdINQd27/v53Tg6cerx2nHqGCPQrXt1E79b79Ycy6/S3t07ah9jz04kZN4JKm6apV6lBRtvV9S3F3V1atijCg6aci1e5/PqedXj9dBp/77u/zV2y7WE6PukiTVbd6iRTOe1nE/ODPmSHff5pXvafPK9/TRP+rPy73/6HMquvQ/VbN2vQr361rfTe7XVTXvfRBzpOEl+rvcQoZKQ2vsHGWdpKId7O+efWyH3L3c3fu4e58oi6Qkzbthhqb1vVy/P/GH+uvI36j6mQrNubz+B8vBX++nd/6yQLWf1jRylNZp3vwF6tWrp0pLe6iwsFDDhg3WI48+HndYkUhDrs/dOEOT+12u+7/0Qz0+8jeqeqZCT4y6S+3367L1OT0HnqD33qiML8hAatas16fVa7XnIfU/XjqffLQ2vblC6x6fp32H9Zck7Tusv9Y99mKMUUYjDd/lpGmsoxwt6Ukze1PSiuy+AyX1knRphHEFcfDgE/XP3zwSdxiRqa2t1ajR4zR71lQVZDKaPGW6KioWxx1WJNKU6/ZOv/1iteu2l2TS2oXvaM5Vk+IOKYjl4yaq1x2jZYVt9Ok772rpD++QZUy97/6x9j13gDZXrdHiH/wq7jCDS/R3OaG/Zsu8kXMAZpaR1E/bTuaZ5028YOZ/Soan5iTDxaufijsEROTW/U+NO4S86lO7Me4Q8ubLa1+IO4S82rK5yqI69icvPhj85/2e/f5PZPE2VaOzXt29TtLzeYgFAIAWJzELDgAAYpbGy0MAAEg7OkoAQBgpvTwEAIBUo6MEAISR0HOUFEoAQBgJLZQMvQIAkAMdJQAgiKT+4mY6SgAAcqCjBACEkdBzlBRKAEAYXEcJAED60FECAMJI6NArHSUAADnQUQIAwkjoOUoKJQAgDIZeAQBIHzpKAEAYCR16paMEACAHOkoAQBicowQAIH3oKAEAYSS0o6RQAgDCYDIPAADpQ0cJAAgjoUOvdJQAAORARwkACCOh5ygplACAMBh6BQAgfegoAQBhJHTolY4SAIAcIu8oL179VNRv0WJ8/PLkuEPIqw7HfyfuEPJm9Lvp+R4DzZbQc5QMvQIAwkhooWToFQCAHCiUAIAw3MNvTWBmk8xstZm91mDf3mb2hJm9mf2za3a/mdntZrbEzF4xs+MbOz6FEgDQ2k2WdMZ2+8ZKetLde0t6Mntfkr4mqXd2K5N0V2MHp1ACAMKoqwu/NYG7Py3p/e12D5Y0JXt7iqQhDfbf7/Wel9TFzLrnOj6FEgDQYplZmZnNb7CVNfGl+7v7yuztVZL2z94ulrSiwfMqs/t2ilmvAIAwIpj16u7lksp38xhuZk074bkDFEoAQBgta2Wed82su7uvzA6trs7ur5LUo8HzSrL7doqhVwBAEj0s6YLs7QskzWyw/9vZ2a9flPRBgyHaHaKjBACEEdOCA2Y2TVJ/SfuYWaWkayTdIGmGmY2Q9LakYdmnz5Z0pqQlkjZK+m5jx6dQAgBaNXc/bycPDdjBc13SyF05PoUSABBGExcIaG0olACAMFjrFQCA9KGjBACEQUcJAED60FECAMJoWQsOBEOhBAAE4XXJnPXK0CsAADnQUQIAwmAyDwAA6UNHCQAII6GTeegoAQDIgY4SABBGQme9UigBAGEwmQcAgPShowQAhEFHCQBA+tBRAgDC4Bc3AwCQA0OvAACkT+IK5cDT+2vha0/r9Yq5GnPlyLjDCeK/fvNbfeW7YzV09Pit+x5/9mUNHfVLHfPNy7Rwydufec3KNe/rC9+6QpNn/iWfoUYqiZ9tLuSbXInNtc7Dby1AogplJpPR7beN16CzhuuoY07VOecM0eGH9447rN12dv8v6q6fbfs/U68DizRhzIU64YhDdviamyb/SV8+7sh8hJcXSf1sd4Z8k5tvmnJNikQVyn59j9PSpcv11lvvqKamRjNmzNTZZw2MO6zd1ufIXurcsf02+w4uOUA9i/ff4fP/+sI/VbxfNx3S44B8hJcXSf1sd4Z8k5tvonP1uvBbC9DsQmlm3w0ZSAhFxQdoRWX11vuVVStVVJScYtEUGzd9qkl/fkIXDzsz7lCCSttnS77JzTfRuTL0+hn/HSwKBHPnjFk6f9Bpat9uj7hDAYBEyHl5iJm9srOHJO143K/+dWWSyiTJCjork+nQ7AB3RXXVKvUoKdp6v6S4u6qrV+XlvVuKV998W395boFueeDP+vDjTbKMaY/CQp135lfiDm23pO2zJd/k5pvkXD2hl4c0dh3l/pIGSlq33X6T9OzOXuTu5ZLKJalN2+K89c7z5i9Qr149VVraQ1VVqzRs2GCd/+0EzShrgim//OHW23dOn6X2e+7R6ouklL7PlnyTm2+ack2Kxgrlo5I6uvuC7R8wszlRBLQ7amtrNWr0OM2eNVUFmYwmT5muiorFcYe128ZMuE/zF76p9R9+pP+4cJwuOedMde7UQddPfFDrNnykkdfdrcNKi3X3f10ad6iRSepnuzPkm9x8E51rCzmnGJp5xEsO5bOjjNvHL0+OO4S86nD8d+IOAcAu2rK5yqI69sfjvx38532Hq++PLN6mYgk7AEAYLeRyjtAolACAMBI69JqoBQcAAAiNjhIAEEZCLw+howQAIAc6SgBAGAk9R0mhBACEkdBZrwy9AgCQAx0lACCMhA690lECAJADHSUAIIi0/vYQAACahqFXAADSh44SABAGHSUAAOlDRwkACIMFBwAASB86SgBAGAk9R0mhBAAE4QktlAy9AgCQAx0lACAMOkoAANKHjhIAEAZrvQIAkANDrwAApA8dJQAgDDpKAADSh44SABCEezI7SgolACAMhl4BAEgfOkoAQBgJ7SgplAEVf2lk3CHk1cals+MOIW/aH3Jm3CEAiAmFEgAQBL89BACAFsjMfmhmC83sNTObZmZ7mllPM3vBzJaY2XQza9vc41MoAQBh1Hn4rRFmVizpckl93P3zkgoknSvpRkm3uHsvSeskjWhuWhRKAEAYdRFsTdNGUjszayOpvaSVkk6T9Ifs41MkDWluWhRKAECr5e5Vkm6W9I7qC+QHkl6StN7dt2SfVimpuLnvQaEEAAThdR58M7MyM5vfYCtr+J5m1lXSYEk9JRVJ6iDpjJB5MesVANBiuXu5pPIcT/kPSW+5+xpJMrM/STpJUhcza5PtKkskVTU3BjpKAEAYMUzmUf2Q6xfNrL2ZmaQBkiokPSXpm9nnXCBpZnPTolACAMKIYTKPu7+g+kk7L0t6VfV1rVzSTyRdYWZLJHWTdG9z02LoFQDQqrn7NZKu2W73Mkn9QhyfQgkACIKVeQAASCE6SgBAGE1fIKBVoVACAIJg6BUAgBSiowQAhJHQoVc6SgAAcqCjBAAE4QntKCmUAIAwElooGXoFACAHOkoAQBBJHXqlowQAIAc6SgBAGHSUAACkDx0lACCIpJ6jpFACAIJIaqFk6BUAgBzoKAEAQdBRAgCQQnSUAIAw3OKOIBIUSgBAEAy9thIDT++vha89rdcr5mrMlSPjDidye3XupEn3367n5v+vnp33/9Sn37Fxh7TbfnbTnfrKN7+vod//0dZ9j/3tOQ0ZcYWO/uo5WvjG0q37a2q2aNxNd2ro93+kb5RdqXkLFsYRciTS9l1OU75pyjUJElUoM5mMbr9tvAadNVxHHXOqzjlniA4/vHfcYUXquhvH6a9/+btO7HOGvvKls7W4QRFprQYP7K+7rv/pNvt6l/bQLT//sU446vBt9v9h9l8kSQ9N/JXKbxynm+65X3V1rf+ftWn7Lqcp3yTn6nUWfGsJElUo+/U9TkuXLtdbb72jmpoazZgxU2efNTDusCLTaa+OOvFLffTb+x+UJNXU1GjDBx/GHNXu63P0EercqeM2+w4+qEQ9exR95rlL367UF479vCSpW9fO2qtjBy1cvCwvcUYpbd/lNOWbplyTotFCaWaHmdkAM+u43f4zogureYqKD9CKyuqt9yurVqqo6IAYI4rWQQf10HvvrdOv77pBf/37n3Xrr8erfft2cYeVV4ceXKqnnpuvLbW1qly5WhWLl2nV6rVxh7Xb0vZdTlO+Sc7V68JvLUHOQmlml0uaKekySa+Z2eAGD18XZWBoXJs2BTr6mCN0371TddrJQ/Txxo26/IqyuMPKq6FfO1X777O3zr1krG68c7KOOfJQZQoSNVACtBruFnxrCRqb9XqhpBPc/SMzK5X0BzMrdffbJO00AzMrk1QmSVbQWZlMh1Dx5lRdtUo9Sv49PFdS3F3V1avy8t5xqK5apeqqVXp5/iuSpEf+/JhGpaxQtiko0E8u+c7W+8MvH6fSks8O0bY2afwupyXfNOWaFI390zvj7h9Jkrsvl9Rf0tfMbIJyFEp3L3f3Pu7eJ19FUpLmzV+gXr16qrS0hwoLCzVs2GA98ujjeXv/fFu9eq2qqlapV6+ekqRT+p+oN15fEnNU+bXpk0+1cdMnkqRnX3pFBQUFOuSgkpij2n1p+y6nKd8k55rUodfGOsp3zexYd18gSdnOcpCkSZKOijq4XVVbW6tRo8dp9qypKshkNHnKdFVULI47rEhddeW1unvizSpsW6i3l1fqskvGxh3Sbhsz/lbN+2eF1n/woQace5FGXjBMnTt11HV3TNK6Dzbokqtv0GGHlOqeG6/W++s/0EVjx8syGe3XbW9dP/bSuMMPIm3f5TTlm6Zck8LcfecPmpVI2uLunxkXMLOT3P2Zxt6gTdvinb9BwnTZM3/dc0tQvfDBuEPIm/aHnBl3CEAQWzZXRXbib0XfAcF/3veY92TsJypzdpTuXpnjsUaLJAAArR1L2AEAgsgxQNmqUSgBAEG0lJV0QuOCMwAAcqCjBAAEQUcJAEAK0VECAIJgMg8AADkw9AoAQArRUQIAgmgpv+0jNDpKAAByoKMEAATRUn7bR2gUSgBAEHUMvQIAkD50lACAIJjMAwBACtFRAgCCYMEBAABSiI4SABAEa70CAJADQ68AAKQQHSUAIAgWHAAAIIXoKAEAQSR1wQEKJQAgiKTOemXoFQCAHOgoAQBBMJkHAIAUoqMEAATBZB4AAHJgMg8AAClERwkACCKpk3kolAGt/+TjuEPIq/aHnBl3CHmzqfrvcYeQV+2KTo47BKDJzKyLpImSPi/JJX1P0huSpksqlbRc0jB3X9ec4zP0CgAIwt2Cb010m6T/dffDJB0jaZGksZKedPfekp7M3m8WCiUAoNUys86STpF0ryS5+2Z3Xy9psKQp2adNkTSkue/B0CsAIIiYzlH2lLRG0n1mdoyklySNkrS/u6/MPmeVpP2b+wZ0lACAIDyCzczKzGx+g61su7dtI+l4SXe5+3GSPtZ2w6zu/q/DNQsdJQCgxXL3cknlOZ5SKanS3V/I3v+D6gvlu2bW3d1Xmll3SaubGwMdJQAgiDq34Ftj3H2VpBVmdmh21wBJFZIelnRBdt8FkmY2Ny86SgBAa3eZpN+ZWVtJyyR9V/WN4AwzGyHpbUnDmntwCiUAIIi41np19wWS+uzgoQEhjk+hBAAEURd3ABHhHCUAADnQUQIAgnAlc61XOkoAAHKgowQABFGX0N9HSaEEAARRx9ArAADpQ0cJAAiCyTwAAKQQHSUAIAgWHAAAIIXoKAEAQST1HCWFEgAQBEOvAACkEB0lACAIOkoAAFKIjhIAEASTeQAAyKEumXWSoVcAAHKhowQABMFvDwEAIIXoKAEAQST09zZTKAEAYXAdZSsx8PT+Wvja03q9Yq7GXDky7nAil6Z8k5jruOsm6JSvn6shwy/auu/mOybqrPMu1NBvX6zLr/qFNnz4kSTp2Rdf1rDvXaah51+sYd+7TC+8tCCmqKORxM93Z9KUaxKYe7TNcpu2xXnrxjOZjBYt/LvOOPM8VVau1PPPzdbw8y/RokVv5iuEvEpTvnHnuqn675Ecd/6CV9W+XTv99Nqb9eff3i1JeuaFl/SFE45VmzYFmnDnvZKkKy4ZoUWLl6hb167ab99uenPZcv3gh+P015m/jSSudkUnR3LcnYn7882nuHPdsrkqshk3f+j+reA/77+58nexzxBqtKM0s35m1jd7+wgzu8LMzow+tF3Xr+9xWrp0ud566x3V1NRoxoyZOvusgXGHFZk05ZvUXPsce5Q679Vpm30nfeEEtWlTIEk6+sjD9O7qtZKkwz/XS/vt202S1KvnQfrk00+1efPm/AYckaR+vjuSplyTImehNLNrJN0u6S4zu17SHZI6SBprZlfnIb5dUlR8gFZUVm+9X1m1UkVFB8QYUbTSlG+acm3ooVmP68sn9v3M/ifmzNURh/ZS27ZtY4gqvDR9vknO1SPYWoLGJvN8U9KxkvaQtEpSibtvMLObJb0gaXy04QHpdc+UaSooKNCg00/dZv+SZW9rwp2TVH4L//sB+dBYodzi7rWSNprZUnffIEnuvsnMdjrByczKJJVJkhV0VibTIVjAuVRXrVKPkqKt90uKu6u6elVe3jsOaco3TblK0p9nPaGnn3lRE2+/Xmb/PkWzavUajfrptbruZz/WgQ3+Plq7NH2+Sc41rbNeN5tZ++ztE/6108w6K8ffibuXu3sfd++TryIpSfPmL1CvXj1VWtpDhYWFGjZssB559PG8vX++pSnfNOU69/n5mjT1Qf36xmvUbs89t+7f8OFHuuTKazT6ou/q+KOPjDHC8NL0+SY51zoLv7UEjXWUp7j7p5Lk7g0LY6GkCyKLqplqa2s1avQ4zZ41VQWZjCZPma6KisVxhxWZNOWb1FyvvOYGzfvHK1q/foMGDBmuS0acr4kPTNfmmhpdOLp+GsDRRx6ma8Zcpml/fEQrKqt1931Tdfd9UyVJ5beOV7euXWLMIIykfr47kqZckyJRl4cAUYnq8pCWKt+XhyB/orw85HdFw4P/vP9W9W9j7ysTt+AAAAAhsYQdACCIpA4fUigBAEG0lMk3oTH0CgBADnSUAIAg0nodJQAAqUZHCQAIgsk8AADkwGQeAABSiI4SABAEk3kAAEghOkoAQBB0lAAApBAdJQAgCE/orFcKJQAgCIZeAQBIITpKAEAQdJQAAKQQHSUAIAjWegUAIAfWegUAIIXoKAEAQTCZBwCAFKKjBAAEkdSOkkIJAAgiqbNeGXoFACAHOkoAQBBcHgIAQArRUQIAgkjqZB46SgAAcqBQAgCC8Ai2pjKzAjP7h5k9mr3f08xeMLMlZjbdzNo2Ny+GXoEmaFd0ctwh5NWm6r/HHULepO2zjVJdvBeIjJK0SNJe2fs3SrrF3X9vZndLGiHpruYcmI4SANCqmVmJpK9Lmpi9b5JOk/SH7FOmSBrS3OPTUQIAgohxMs+tksZI6pS9303Senffkr1fKam4uQenowQAtFhmVmZm8xtsZds9PkjSand/KaoY6CgBAEFEcYbS3cslled4ykmSzjazMyXtqfpzlLdJ6mJmbbJdZYmkqubGQEcJAAiiLoKtMe5+lbuXuHuppHMl/dXdvyXpKUnfzD7tAkkzm5sXhRIAkEQ/kXSFmS1R/TnLe5t7IIZeAQBBxL3Wq7vPkTQne3uZpH4hjktHCQBADnSUAIAgYl5wIDIUSgBAEMkskwy9AgCQEx0lACAIfs0WAAApREcJAAiCyTwAAOSQzDLJ0CsAADnRUQIAgmAyDwAAKURHCQAIIqmTeegoAQDIgY4SABBEMvtJCiUAIBAm8wAAkEJ0lACAIDyhg690lAAA5EBHCQAIIqnnKCmUAIAguI4SAIAUoqMEAASRzH6SjhIAgJzoKAEAQXCOspUYeHp/LXztab1eMVdjrhwZdziRS1O+acpVSma+466boFO+fq6GDL9o676b75ios867UEO/fbEuv+oX2vDhR5KkZ198WcO+d5mGnn+xhn3vMr3w0oKYog4viZ+tVD/rNfTWEiSqUGYyGd1+23gNOmu4jjrmVJ1zzhAdfnjvuMOKTJryTVOuUnLzHXLmV3X3hF9us+/EvsfpoQfu1kP336XSHsWa+MB0SVLXLnvpjht/roceuEvjx/1IV/3i5jhCDi6pn22SJapQ9ut7nJYuXa633npHNTU1mjFjps4+a2DcYUUmTfmmKVcpufn2OfYodd6r0zb7TvrCCWrTpkCSdPSRh+nd1WslSYd/rpf227ebJKlXz4P0yaefavPmzfkNOAJJ/Wyl+pV5Qv/XEuxyoTSz+6MIJISi4gO0orJ66/3KqpUqKjogxoiilaZ805SrlL58/+WhWY/ryyf2/cz+J+bM1RGH9lLbtm1jiCqstH62rVnOyTxm9vD2uySdamZdJMndz44oLgApc8+UaSooKNCg00/dZv+SZW9rwp2TVH7L+JgiQ1O1lHOKoTU267VEUoWkiaq/RMYk9ZH0q1wvMrMySWWSZAWdlcl02P1Im6C6apV6lBRtvV9S3F3V1avy8t5xSFO+acpVSl++f571hJ5+5kVNvP16mdnW/atWr9Gon16r6372Yx3Y4O+jNUvbZ5sEjQ299pH0kqSrJX3g7nMkbXL3v7n733b2Incvd/c+7t4nX0VSkubNX6BevXqqtLSHCgsLNWzYYD3y6ON5e/98S1O+acpVSle+c5+fr0lTH9Svb7xG7fbcc+v+DR9+pEuuvEajL/qujj/6yBgjDCvJn21Sz1Hm7CjdvU7SLWb2YPbPdxt7TZxqa2s1avQ4zZ41VQWZjCZPma6KisVxhxWZNOWbplyl5OZ75TU3aN4/XtH69Rs0YMhwXTLifE18YLo219TowtFXS6qf0HPNmMs07Y+PaEVlte6+b6ruvm+qJKn81vHq1rVLjBnsvqR+tlJyh17NvekV28y+Lukkd/9pU1/Tpm1xy/gnAYAm21T997hDyJt2RSfHHUJebdlcZY0/q3kuKP1G8J/3U5b/MbJ4m2qXukN3nyVpVkSxAABasbpdaLxak0RdRwkAQGgt9nwjAKB1SWY/SaEEAATCougAAKQQHSUAIIiWct1jaHSUAADkQEcJAAgiqQsOUCgBAEEwmQcAgBSiowQABMFkHgAAUoiOEgAQRFIn89BRAgCQAx0lACCIXfm1ja0JhRIAEASXhwAAkEJ0lACAIJjMAwBACtFRAgCCSOqCAxRKAEAQTOYBACCF6CgBAEEk9TpKOkoAAHKgowQABJHUy0MolACAIJI665WhVwAAcqCjBAAEweUhAACkEB0lACAILg8BAKCFMbMeZvaUmVWY2UIzG5Xdv7eZPWFmb2b/7Nrc96BQAgCCqJMH35pgi6QfufsRkr4oaaSZHSFprKQn3b23pCez95uFodeAMmZxh5BXdQkdZoHUrujkuEPIm41LZ8cdQmLEcXmIu6+UtDJ7+0MzWySpWNJgSf2zT5siaY6knzTnPegoAQCJYGalko6T9IKk/bNFVJJWSdq/ucelowQABBHFKJOZlUkqa7Cr3N3Ld/C8jpL+KGm0u2+wBiN87u5m1uzgKJQAgBYrWxQ/UxgbMrNC1RfJ37n7n7K73zWz7u6+0sy6S1rd3BgYegUABOERbI2x+tbxXkmL3H1Cg4celnRB9vYFkmY2Ny86SgBAEDGtzHOSpPMlvWpmC7L7firpBkkzzGyEpLclDWvuG1AoAQCtlrvPlbSzSw4GhHgPCiUAIAjWegUAIIXoKAEAQSR1rVcKJQAgCIZeAQBIITpKAEAQcaz1mg90lAAA5EBHCQAIIqmTeegoAQDIgY4SABBEUme9UigBAEEw9AoAQArRUQIAgkjq0CsdJQAAOdBRAgCCSOqCAxRKAEAQdUzmAQAgfegoAQBBJHXolY4SAIAc6CgBAEEk9RwlhRIAEARDrwAApBAdJQAgiKQOvdJRAgCQAx0lACAIzlG2EgNP76+Frz2t1yvmasyVI+MOJ1Ll99ysyhUL9I+X/xJ3KHmRps9WIt/W7mc33amvfPP7Gvr9H23d99jfntOQEVfo6K+eo4VvLN26v6Zmi8bddKeGfv9H+kbZlZq3YGEcIWMnElUoM5mMbr9tvAadNVxHHXOqzjlniA4/vHfcYUXm/gce1KCzhscdRl6k7bMl39af7+CB/XXX9T/dZl/v0h665ec/1glHHb7N/j/Mrv/H7kMTf6XyG8fppnvuV11dXd5iDaXOPfjWEuxSoTSzL5vZFWZ2elQB7Y5+fY/T0qXL9dZb76impkYzZszU2WcNjDusyMyd+4LWrVsfdxh5kbbPlnxbf759jj5CnTt13GbfwQeVqGePos88d+nblfrCsZ+XJHXr2ll7deyghYuX5SXOkDyC/1qCnIXSzF5scPtCSXdI6iTpGjMbG3Fsu6yo+ACtqKzeer+yaqWKig6IMSKEkrbPlnyTne/2Dj24VE89N19bamtVuXK1KhYv06rVa+MOC1mNTeYpbHC7TNJX3X2Nmd0s6XlJN0QWGQCkxNCvnapl71Tq3EvGqvt+++qYIw9VpqD1nRlzb33DxU3RWKHMmFlX1Xee5u5rJMndPzazLTt7kZmVqb6wygo6K5PpECrenKqrVqlHyb+HNUqKu6u6elVe3hvRSttnS77Jznd7bQoK9JNLvrP1/vDLx6m05LNDtIhHY/9k6SzpJUnzJe1tZt0lycw6SrKdvcjdy929j7v3yVeRlKR58xeoV6+eKi3tocLCQg0bNliPPPp43t4f0UnbZ0u+yc53e5s++VQbN30iSXr2pVdUUFCgQw4qiTmqXVcnD761BDk7Sncv3clDdZKGBo9mN9XW1mrU6HGaPWuqCjIZTZ4yXRUVi+MOKzIP3H+HTjnlRO2zz95atnSefnHtrzR58u/jDisSaftsybf15ztm/K2a988Krf/gQw049yKNvGCYOnfqqOvumKR1H2zQJVffoMMOKdU9N16t99d/oIvGjpdlMtqv2966fuylcYffLN5CZqmGZlEn1qZtcTL/5nYgYzttshOppUzdBnbHxqWz4w4hr9r2OCayH1QH7n1U8B8K77z/auw/WFmZBwAQREsZKg2t9U2rAgAgj+goAQBBJPUcJYUSABBEUuctMPQKAEAOdJQAgCBaytqsodFRAgCQAx0lACCIpE7moaMEACAHOkoAQBBJXXCAQgkACIKhVwAAUoiOEgAQBAsOAACQQnSUAIAgknqOkkIJAAgiqbNeGXoFACAHOkoAQBBJHXqlowQAIAc6SgBAEEm9PIRCCQAIgl+zBQBACtFRAgCCSOrQKx0lAAA50FECAILg8hAAAFKIjhIAEERSZ71SKAEAQTD0CgBAC2RmZ5jZG2a2xMzGhj4+HSUAIIg4OkozK5D0G0lflVQpaZ6ZPezuFaHeg44SANCa9ZO0xN2XuftmSb+XNDjkG1AoAQBBeARbExRLWtHgfmV2XzCRD71u2VxlUb/HjphZmbuXx/He+ZamXCXyTbI05SolL98oft6bWZmksga7yvP9d5bkjrKs8ackRppylcg3ydKUq5S+fHeZu5e7e58G2/ZFskpSjwb3S7L7gklyoQQAJN88Sb3NrKeZtZV0rqSHQ74Bs14BAK2Wu28xs0slPSapQNIkd18Y8j2SXCgTM+7fBGnKVSLfJEtTrlL68o2Eu8+WNDuq41tSV1IAACAEzlECAJBD4gpl1EsZtSRmNsnMVpvZa3HHkg9m1sPMnjKzCjNbaGaj4o4pKma2p5m9aGb/zOb633HHFDUzKzCzf5jZo3HHEjUzW25mr5rZAjObH3c8yC1RQ6/ZpYwWq8FSRpLOC7mUUUtiZqdI+kjS/e7++bjjiZqZdZfU3d1fNrNOkl6SNCSJn6+ZmaQO7v6RmRVKmitplLs/H3NokTGzKyT1kbSXuw+KO54omdlySX3cfW3csaBxSesoI1/KqCVx96clvR93HPni7ivd/eXs7Q8lLVLgFThaCq/3UfZuYXZLzr9qt2NmJZK+Lmli3LEA20taoYx8KSO0DGZWKuk4SS/EHEpkskORCyStlvSEuyc2V0m3ShojqS7mOPLFJT1uZi9lV55BC5a0QokUMLOOkv4oabS7b4g7nqi4e627H6v6lUb6mVkih9fNbJCk1e7+Utyx5NGX3f14SV+TNDJ7GgUtVNIKZeRLGSFe2fN1f5T0O3f/U9zx5IO7r5f0lKQzYg4lKidJOjt73u73kk4zs9/GG1K03L0q++dqSQ+p/rQRWqikFcrIlzJCfLITXO6VtMjdJ8QdT5TMbF8z65K93U71E9RejzWoiLj7Ve5e4u6lqv9/9q/uPjzmsCJjZh2yk9FkZh0knS4pFTPXW6tEFUp33yLpX0sZLZI0I/RSRi2JmU2T9JykQ82s0sxGxB1TxE6SdL7qO44F2e3MuIOKSHdJT5nZK6r/B+AT7p74yyZSYn9Jc83sn5JelDTL3f835piQQ6IuDwEAILREdZQAAIRGoQQAIAcKJQAAOVAoAQDIgUIJAEAOFEoAAHKgUAIAkAOFEgCAHP4/n3YBQX7NxSwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_matrix = confusion_matrix(y_test, prediction)\n",
    "\n",
    "plt.figure(figsize=(8, 8))\n",
    "sns.heatmap(conf_matrix, annot=True, fmt='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}