jupyter_examples/MNIST_optuna_miha.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of the efficiency of the MIHA algorithm with the Optuna framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "import optuna\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import cm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model obtained using the MIHA algorithm\n",
    "\n",
    "The number of epochs produced that it took to train the model is 176."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import jit\n",
    "\n",
    "net = jit.load('D:/miha_exp/mnist/4/model/model_final.zip')\n",
    "state = torch.load('D:/miha_exp/mnist/4/model/optimizer_final.pth')\n",
    "\n",
    "# Uploading 10 000 MNIST objects for training\n",
    "X_train = torch.load('D:/ITMO/Neural_network_example/mnist/X_train.pt')\n",
    "Y_train = torch.load('D:/ITMO/Neural_network_example/mnist/Y_train.pt')\n",
    "\n",
    "# And for testing\n",
    "X_test = torch.load('D:/ITMO/Neural_network_example/mnist/X_test.pt')\n",
    "Y_test = torch.load('D:/ITMO/Neural_network_example/mnist/Y_test.pt')\n",
    "\n",
    "# We check accuracy on 500 objects\n",
    "X_test = X_test[:500, :]\n",
    "Y_test = Y_test[:500, ]\n",
    "\n",
    "train = torch.utils.data.TensorDataset(X_train, Y_train)\n",
    "test = torch.utils.data.TensorDataset(X_test, Y_test)\n",
    "\n",
    "# Prepare data loaders\n",
    "train_loader = torch.utils.data.DataLoader(train, batch_size=20, num_workers=0)\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=20, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check the accuracy of the model on the test dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy - 0.974\n",
      "487 labels out of 500 objects were correctly assigned\n"
     ]
    }
   ],
   "source": [
    "def validation(net, test_loader):\n",
    "    net.eval()\n",
    "    \n",
    "    true_labels = []\n",
    "    predicted_labels = []\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            # The probability of assignment to classes\n",
    "            output = net(data)\n",
    "            # Predicted labels\n",
    "            pred = output.argmax(dim=1, keepdim=False)\n",
    "            \n",
    "            # Number of correctly predicted class labels\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "            \n",
    "            # Transform to lists\n",
    "            target = list(map(int, target))\n",
    "            pred = list(map(int, pred))\n",
    "            \n",
    "            true_labels.extend(list(target))\n",
    "            predicted_labels.extend(list(pred))\n",
    "\n",
    "        accuracy = correct / len(test_loader.dataset)\n",
    "   \n",
    "    print(f'Accuracy - {accuracy}')\n",
    "    print(f'{correct} labels out of {len(test_loader.dataset)} objects were correctly assigned')\n",
    "    \n",
    "validation(net = net, test_loader = test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thus, the prepared neural network made a mistake only on 13 objects.\n",
    "\n",
    "## Feedforward neural network (FNN) which was optimized by Optuna "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimization using the optuna framework. The code below is a modified version of the official example, see [here](https://github.com/optuna/optuna/blob/master/examples/pytorch_simple.py)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:32:49,129]\u001b[0m A new study created in memory with name: no-name-9a9f59f3-225b-4267-a0fe-660d351169b6\u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:33:07,805]\u001b[0m Trial 0 finished with value: 0.9098 and parameters: {'n_layers': 3, 'n_units_l0': 429, 'dropout_l0': 0.41084891219713937, 'n_units_l1': 85, 'dropout_l1': 0.22274557247090618, 'n_units_l2': 276, 'dropout_l2': 0.25610214111637236, 'optimizer': 'Adam', 'lr': 5.736254519743909e-05}. Best is trial 0 with value: 0.9098.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:33:16,182]\u001b[0m Trial 1 finished with value: 0.2741 and parameters: {'n_layers': 3, 'n_units_l0': 64, 'dropout_l0': 0.2297348311081977, 'n_units_l1': 328, 'dropout_l1': 0.33050124281172644, 'n_units_l2': 483, 'dropout_l2': 0.3402349211188188, 'optimizer': 'SGD', 'lr': 0.0007298508269815263}. Best is trial 0 with value: 0.9098.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:33:19,942]\u001b[0m Trial 2 finished with value: 0.3112 and parameters: {'n_layers': 1, 'n_units_l0': 94, 'dropout_l0': 0.34956241407144784, 'optimizer': 'SGD', 'lr': 8.472899768329822e-05}. Best is trial 0 with value: 0.9098.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:33:24,798]\u001b[0m Trial 3 finished with value: 0.4984 and parameters: {'n_layers': 1, 'n_units_l0': 12, 'dropout_l0': 0.4304600844849822, 'optimizer': 'Adam', 'lr': 2.5603264038579248e-05}. Best is trial 0 with value: 0.9098.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:33:35,963]\u001b[0m Trial 4 finished with value: 0.7508 and parameters: {'n_layers': 1, 'n_units_l0': 201, 'dropout_l0': 0.30385189404091856, 'optimizer': 'Adam', 'lr': 0.03264575763193049}. Best is trial 0 with value: 0.9098.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:33:53,481]\u001b[0m Trial 5 finished with value: 0.9667 and parameters: {'n_layers': 1, 'n_units_l0': 401, 'dropout_l0': 0.48907092040477557, 'optimizer': 'Adam', 'lr': 0.002289667742601567}. Best is trial 5 with value: 0.9667.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:34:00,360]\u001b[0m Trial 6 finished with value: 0.8905 and parameters: {'n_layers': 2, 'n_units_l0': 200, 'dropout_l0': 0.21142294234329867, 'n_units_l1': 206, 'dropout_l1': 0.40480955681719555, 'optimizer': 'SGD', 'lr': 0.00681814627138274}. Best is trial 5 with value: 0.9667.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:34:01,248]\u001b[0m Trial 7 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:34:11,895]\u001b[0m Trial 8 finished with value: 0.9609 and parameters: {'n_layers': 2, 'n_units_l0': 253, 'dropout_l0': 0.43186054230457743, 'n_units_l1': 289, 'dropout_l1': 0.4670186860605348, 'optimizer': 'RMSprop', 'lr': 0.0015178778941547532}. Best is trial 5 with value: 0.9667.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:34:18,013]\u001b[0m Trial 9 finished with value: 0.9238 and parameters: {'n_layers': 1, 'n_units_l0': 50, 'dropout_l0': 0.43025507477368063, 'optimizer': 'Adam', 'lr': 0.0012490237418554724}. Best is trial 5 with value: 0.9667.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:34:19,424]\u001b[0m Trial 10 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:34:35,858]\u001b[0m Trial 11 finished with value: 0.9666 and parameters: {'n_layers': 2, 'n_units_l0': 352, 'dropout_l0': 0.49253925195981973, 'n_units_l1': 487, 'dropout_l1': 0.4950896635134939, 'optimizer': 'RMSprop', 'lr': 0.0009767757008174829}. Best is trial 5 with value: 0.9667.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:34:53,208]\u001b[0m Trial 12 finished with value: 0.9716 and parameters: {'n_layers': 2, 'n_units_l0': 373, 'dropout_l0': 0.4992637990091811, 'n_units_l1': 472, 'dropout_l1': 0.4018178348296462, 'optimizer': 'RMSprop', 'lr': 0.00033409727928419814}. Best is trial 12 with value: 0.9716.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:35:09,474]\u001b[0m Trial 13 finished with value: 0.9676 and parameters: {'n_layers': 2, 'n_units_l0': 365, 'dropout_l0': 0.4990610086754125, 'n_units_l1': 498, 'dropout_l1': 0.3748502550427031, 'optimizer': 'RMSprop', 'lr': 0.0002067646445850057}. Best is trial 12 with value: 0.9716.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:35:21,099]\u001b[0m Trial 14 finished with value: 0.9658 and parameters: {'n_layers': 2, 'n_units_l0': 315, 'dropout_l0': 0.46775274724032295, 'n_units_l1': 497, 'dropout_l1': 0.3677141381615474, 'optimizer': 'RMSprop', 'lr': 0.00021647237048110147}. Best is trial 12 with value: 0.9716.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:35:33,803]\u001b[0m Trial 15 finished with value: 0.9767 and parameters: {'n_layers': 2, 'n_units_l0': 316, 'dropout_l0': 0.3656487254629422, 'n_units_l1': 428, 'dropout_l1': 0.4172312904515172, 'optimizer': 'RMSprop', 'lr': 0.0003004739095190486}. Best is trial 15 with value: 0.9767.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:35:35,098]\u001b[0m Trial 16 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:35:37,205]\u001b[0m Trial 17 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:35:38,198]\u001b[0m Trial 18 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:35:39,837]\u001b[0m Trial 19 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:35:57,779]\u001b[0m Trial 20 finished with value: 0.9902 and parameters: {'n_layers': 2, 'n_units_l0': 413, 'dropout_l0': 0.24735187090626332, 'n_units_l1': 341, 'dropout_l1': 0.40304196331640596, 'optimizer': 'RMSprop', 'lr': 0.000482214510766566}. Best is trial 20 with value: 0.9902.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:36:13,972]\u001b[0m Trial 21 finished with value: 0.9859 and parameters: {'n_layers': 2, 'n_units_l0': 388, 'dropout_l0': 0.2547200785334911, 'n_units_l1': 340, 'dropout_l1': 0.4200802882142507, 'optimizer': 'RMSprop', 'lr': 0.0003436797847966903}. Best is trial 20 with value: 0.9902.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:36:31,165]\u001b[0m Trial 22 finished with value: 0.9901 and parameters: {'n_layers': 2, 'n_units_l0': 432, 'dropout_l0': 0.2520282086621887, 'n_units_l1': 351, 'dropout_l1': 0.4483788219884629, 'optimizer': 'RMSprop', 'lr': 0.0005839076043443495}. Best is trial 20 with value: 0.9902.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:36:47,990]\u001b[0m Trial 23 finished with value: 0.988 and parameters: {'n_layers': 2, 'n_units_l0': 446, 'dropout_l0': 0.2505264936195003, 'n_units_l1': 340, 'dropout_l1': 0.4630761146981533, 'optimizer': 'RMSprop', 'lr': 0.0005082659723707941}. Best is trial 20 with value: 0.9902.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:36:57,893]\u001b[0m Trial 24 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:37:15,698]\u001b[0m Trial 25 finished with value: 0.9903 and parameters: {'n_layers': 2, 'n_units_l0': 497, 'dropout_l0': 0.20944805967030697, 'n_units_l1': 352, 'dropout_l1': 0.45447755536395607, 'optimizer': 'RMSprop', 'lr': 0.0005672680824260684}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:37:18,183]\u001b[0m Trial 26 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:37:38,300]\u001b[0m Trial 27 finished with value: 0.9821 and parameters: {'n_layers': 3, 'n_units_l0': 493, 'dropout_l0': 0.2870948081280017, 'n_units_l1': 362, 'dropout_l1': 0.44721044676581945, 'n_units_l2': 138, 'dropout_l2': 0.21421046912769587, 'optimizer': 'RMSprop', 'lr': 0.0008151706024498281}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:37:39,740]\u001b[0m Trial 28 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:37:41,622]\u001b[0m Trial 29 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:37:57,983]\u001b[0m Trial 30 finished with value: 0.9797 and parameters: {'n_layers': 2, 'n_units_l0': 486, 'dropout_l0': 0.20049414532674953, 'n_units_l1': 241, 'dropout_l1': 0.3946946489944929, 'optimizer': 'RMSprop', 'lr': 0.0020629250991577347}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:38:14,546]\u001b[0m Trial 31 finished with value: 0.9862 and parameters: {'n_layers': 2, 'n_units_l0': 446, 'dropout_l0': 0.26670926246462545, 'n_units_l1': 358, 'dropout_l1': 0.47082010248726575, 'optimizer': 'RMSprop', 'lr': 0.0006036911748963702}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:38:30,342]\u001b[0m Trial 32 finished with value: 0.9889 and parameters: {'n_layers': 2, 'n_units_l0': 441, 'dropout_l0': 0.23499764952026672, 'n_units_l1': 312, 'dropout_l1': 0.45624965208772844, 'optimizer': 'RMSprop', 'lr': 0.00048783396548309524}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:38:31,418]\u001b[0m Trial 33 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:38:33,054]\u001b[0m Trial 34 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:38:43,579]\u001b[0m Trial 35 finished with value: 0.9886 and parameters: {'n_layers': 1, 'n_units_l0': 342, 'dropout_l0': 0.28237641899177496, 'optimizer': 'RMSprop', 'lr': 0.0012068179395898033}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:38:45,910]\u001b[0m Trial 36 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:38:46,715]\u001b[0m Trial 37 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:39:14,879]\u001b[0m Trial 38 finished with value: 0.976 and parameters: {'n_layers': 2, 'n_units_l0': 500, 'dropout_l0': 0.22266946424950834, 'n_units_l1': 374, 'dropout_l1': 0.2025402841989105, 'optimizer': 'Adam', 'lr': 0.0024268409578525846}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:39:16,033]\u001b[0m Trial 39 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:39:17,134]\u001b[0m Trial 40 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:39:27,787]\u001b[0m Trial 41 finished with value: 0.9901 and parameters: {'n_layers': 1, 'n_units_l0': 364, 'dropout_l0': 0.28194019922838814, 'optimizer': 'RMSprop', 'lr': 0.0012912870826014638}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:39:38,612]\u001b[0m Trial 42 finished with value: 0.9886 and parameters: {'n_layers': 1, 'n_units_l0': 383, 'dropout_l0': 0.23746727010983942, 'optimizer': 'RMSprop', 'lr': 0.0015885145666954168}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:39:52,935]\u001b[0m Trial 43 finished with value: 0.9881 and parameters: {'n_layers': 2, 'n_units_l0': 340, 'dropout_l0': 0.21262723660429034, 'n_units_l1': 285, 'dropout_l1': 0.413580900803986, 'optimizer': 'RMSprop', 'lr': 0.0009089464538772671}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:39:54,090]\u001b[0m Trial 44 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:39:55,561]\u001b[0m Trial 45 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:40:10,774]\u001b[0m Trial 46 finished with value: 0.9883 and parameters: {'n_layers': 2, 'n_units_l0': 369, 'dropout_l0': 0.2948109413321321, 'n_units_l1': 461, 'dropout_l1': 0.3561109397610883, 'optimizer': 'RMSprop', 'lr': 0.0007586623004363177}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:40:25,314]\u001b[0m Trial 47 finished with value: 0.9877 and parameters: {'n_layers': 1, 'n_units_l0': 393, 'dropout_l0': 0.2631760709804497, 'optimizer': 'Adam', 'lr': 0.0015300896321639996}. Best is trial 25 with value: 0.9903.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of epochs produced 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-01-15 22:40:27,059]\u001b[0m Trial 48 pruned. \u001b[0m\n",
      "\u001b[32m[I 2021-01-15 22:40:28,669]\u001b[0m Trial 49 pruned. \u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Study statistics: \n",
      "  Number of finished trials:  50\n",
      "  Number of pruned trials:  20\n",
      "  Number of complete trials:  30\n",
      "Best trial:\n",
      "  Value:  0.9903\n",
      "  Params: \n",
      "    n_layers: 2\n",
      "    n_units_l0: 497\n",
      "    dropout_l0: 0.20944805967030697\n",
      "    n_units_l1: 352\n",
      "    dropout_l1: 0.45447755536395607\n",
      "    optimizer: RMSprop\n",
      "    lr: 0.0005672680824260684\n",
      "\n",
      "Best model accuracy 0.976\n"
     ]
    }
   ],
   "source": [
    "DEVICE = torch.device(\"cpu\")\n",
    "BATCHSIZE = 128\n",
    "CLASSES = 10\n",
    "DIR = os.getcwd()\n",
    "EPOCHS = 10\n",
    "LOG_INTERVAL = 10\n",
    "test_accuracy = 0\n",
    "\n",
    "def define_model(trial):\n",
    "    \n",
    "    # We optimize the number of layers, hidden units and dropout ratio in each layer.\n",
    "    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n",
    "    layers = []\n",
    "\n",
    "    in_features = 28 * 28\n",
    "    for i in range(n_layers):\n",
    "        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 500)\n",
    "        layers.append(nn.Linear(in_features, out_features))\n",
    "        layers.append(nn.ReLU())\n",
    "        p = trial.suggest_float(\"dropout_l{}\".format(i), 0.2, 0.5)\n",
    "        layers.append(nn.Dropout(p))\n",
    "\n",
    "        in_features = out_features\n",
    "    layers.append(nn.Linear(in_features, CLASSES))\n",
    "    layers.append(nn.LogSoftmax(dim=1))\n",
    "    \n",
    "    return nn.Sequential(*layers)\n",
    "\n",
    "def objective(trial):\n",
    "    global test_accuracy\n",
    "\n",
    "    produced_epochs = 0\n",
    "    \n",
    "    # Generate the model.\n",
    "    model = define_model(trial).to(DEVICE)\n",
    "\n",
    "    # Generate the optimizers.\n",
    "    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
    "    lr = trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n",
    "    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)\n",
    "\n",
    "    # Training of the model.\n",
    "    for epoch in range(EPOCHS):\n",
    "        correct = 0\n",
    "        model.train()\n",
    "        for data, target in train_loader:\n",
    "            optimizer.zero_grad()\n",
    "            output = model(data)\n",
    "            \n",
    "            pred = output.argmax(dim=1, keepdim=False)\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "                \n",
    "            loss = F.nll_loss(output, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        \n",
    "        train_accuracy = correct / len(train_loader.dataset)\n",
    "        produced_epochs += 1\n",
    "        \n",
    "        # Validation of the model\n",
    "        model.eval()\n",
    "        correct = 0\n",
    "        with torch.no_grad():\n",
    "            for data, target in test_loader:\n",
    "                output = model(data)\n",
    "                pred = output.argmax(dim=1, keepdim=False)\n",
    "                correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "        new_test_accuracy = correct / len(test_loader.dataset)\n",
    "        \n",
    "        # Update test accuracy\n",
    "        if new_test_accuracy > test_accuracy:\n",
    "            test_accuracy = new_test_accuracy\n",
    "        \n",
    "        trial.report(train_accuracy, epoch)\n",
    "\n",
    "        # Handle pruning based on the intermediate value.\n",
    "        if trial.should_prune():\n",
    "            raise optuna.exceptions.TrialPruned()\n",
    "    \n",
    "    print(f'Number of epochs produced {produced_epochs}')\n",
    "    return train_accuracy\n",
    "\n",
    "def make_test_validation(best_model):\n",
    "    \n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            # Вероятности отнесения к классам\n",
    "            output = best_model(data)\n",
    "            # Предсказанные метки\n",
    "            pred = output.argmax(dim=1, keepdim=False)\n",
    "            \n",
    "            # Количество правильно предсказанных меток класса\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "        accuracy = correct / len(test_loader.dataset)\n",
    "   \n",
    "    print(f'Accuracy - {accuracy}')\n",
    "    print(f'{correct} labels out of {len(test_loader.dataset)} objects were correctly assigned')\n",
    "    \n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction=\"maximize\")\n",
    "    study.optimize(objective, n_trials=50, timeout=600)\n",
    "\n",
    "    pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]\n",
    "    complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]\n",
    "\n",
    "    print(\"Study statistics: \")\n",
    "    print(\"  Number of finished trials: \", len(study.trials))\n",
    "    print(\"  Number of pruned trials: \", len(pruned_trials))\n",
    "    print(\"  Number of complete trials: \", len(complete_trials))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "    \n",
    "    print(\"  Value: \", trial.value)\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "        \n",
    "    print(f'\\nBest model accuracy {test_accuracy}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### As you can see, the model obtained using the MIHA algorithm gave a similar result (accuracy 0.974) to the neural network optimized using Optuna (accuracy 0.976)\n",
    "\n",
    "But the repository with Optuna has several thousand stars, and this library has none, change it (We really tried very hard)!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}