playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.torch_core import *\n",
    "from fastai.data import DataBunch\n",
    "from fastai.callback import *\n",
    "from fastai.basic_train import Learner, LearnerCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import * \n",
    "from fastai.docs import * \n",
    "from fastai.vision import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# __all__ = ['TerminateOnNaN', 'EarlyStopping', 'SaveModel']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('../data/mnist_sample')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "untar_data(MNIST_PATH)\n",
    "MNIST_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = image_data_from_folder(MNIST_PATH, ds_tfms=(rand_pad(2, 28), []))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "\n",
    "The callbacks bellow are based on Keras Callbacks of same name:\n",
    "https://github.com/keras-team/keras/blob/master/keras/callbacks.py\n",
    "\n",
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Problem: Loss is Nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=2), HTML(value='0.00% [0/2 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:08\n",
      "epoch  train loss  valid loss  accuracy\n",
      "0      nan         nan         0.495584  (00:04)\n",
      "1      nan         nan         0.495584  (00:03)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy])\n",
    "learn.fit(2,1e4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Callback bellow is *very* influenced by Keras Callback of same name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TerminateOnNaN(LearnerCallback):\n",
    "    \"A `LearnerCallback` that terminates training if loss is NaN.\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__(learn)\n",
    "        self.stop = False\n",
    "    \n",
    "    def on_batch_end(self,epoch:int, num_batch:int,  **kwargs:Any)->None:\n",
    "        loss = learn.recorder.losses[-1:][0]\n",
    "        if loss is not None:\n",
    "            if torch.isnan(loss):\n",
    "                print (f'Epoch/Batch ({epoch}/{num_batch}): Invalid loss, terminating training.')\n",
    "                self.stop = True\n",
    "                return True\n",
    "    \n",
    "    def on_epoch_end(self, metrics, **kwargs:Any)->None:\n",
    "        return self.stop\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e655b518dbf849c88eb58791ef778ade",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=2), HTML(value='0.00% [0/2 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch/Batch (0/6): Invalid loss, terminating training.\n",
      "Epoch/Batch (0/7): Invalid loss, terminating training.\n"
     ]
    }
   ],
   "source": [
    "learn.fit(2,1e4, callbacks=[TerminateOnNaN()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Actually, I don't know why it went to the second batch if I set return `True` in the first."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Problem: Metric does not improve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=3), HTML(value='0.00% [0/3 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:12\n",
      "epoch  train loss  valid loss  accuracy\n",
      "0      1.036479    0.927704    0.396467  (00:04)\n",
      "1      1.021361    0.945106    0.377821  (00:03)\n",
      "2      1.049802    0.957063    0.383710  (00:03)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(3,1e-42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Callback bellow is basically a simplified port of Keras Early Stopping callback to fastai/pytorch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping(LearnerCallback):\n",
    "    \"A `LearnerCallback` that terminates training when monitored quantity stops improving.\"\n",
    "    def __init__(self, \n",
    "                 monitor='val_loss',\n",
    "                 min_delta=0,\n",
    "                 patience=0,\n",
    "                 mode='auto'):\n",
    "        \n",
    "        super().__init__(learn)\n",
    "        \n",
    "        self.monitor = monitor\n",
    "        self.min_delta = min_delta\n",
    "        self.patience = patience\n",
    "        self.mode = mode\n",
    "        if mode not in ['auto', 'min', 'max']:\n",
    "            #should I use warning?\n",
    "            print(f'TerminateEarly mode {mode} is invalid, falling back to \"auto\" mode.')\n",
    "            mode = 'auto'\n",
    "        if mode == 'min':\n",
    "            self.operator = np.less\n",
    "        elif mode == 'max':\n",
    "            self.operator = np.greater\n",
    "        else:\n",
    "            if 'loss' in self.monitor:\n",
    "                self.operator = np.less\n",
    "            else:\n",
    "                self.operator = np.greater\n",
    "        if self.operator == np.less:\n",
    "            self.min_delta *= -1\n",
    "    \n",
    "    def on_train_begin(self, **kwargs:Any)->None:\n",
    "        self.wait = 0\n",
    "        self.stopped_epoch = 0\n",
    "        self.best = float('inf') if self.operator == np.less else -float('inf')\n",
    "\n",
    "    def on_epoch_end(self, epoch, **kwargs:Any)->None:\n",
    "        current = self.get_monitor_value()\n",
    "        if current is None:\n",
    "            return\n",
    "         \n",
    "        if self.operator(current - self.min_delta, self.best):\n",
    "            \n",
    "            self.best = current\n",
    "            self.wait = 0\n",
    "        else:\n",
    "            \n",
    "            self.wait += 1\n",
    "            if self.wait >= self.patience:\n",
    "                self.stopped_epoch = epoch\n",
    "                return True\n",
    "        return False\n",
    "                \n",
    "    def on_train_end(self, **kwargs:Any)->None:\n",
    "        if self.stopped_epoch > 0:\n",
    "            print(f'Epoch {self.stopped_epoch}: early stopping')\n",
    "                                                                          \n",
    "    def get_monitor_value(self):\n",
    "        values = {'val_loss':learn.recorder.val_losses[-1:][0]\n",
    "               ,'trn_loss':learn.recorder.losses[-1:][0].cpu().numpy()}\n",
    "        for i, name in enumerate(learn.recorder.names[3:]):\n",
    "            values[name]=learn.recorder.metrics[-1:][0][i] \n",
    "\n",
    "        if values.get(self.monitor) is None:\n",
    "            print(f'Early stopping conditioned on metric `{self.monitor}` which is not available. Available metrics are: {\", \".join(map(str, learn.recorder.names[3:]))}')   \n",
    "        return values.get(self.monitor)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy], callbacks=[EarlyStopping(monitor='accuracy', min_delta=0.01, patience=3)] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "493d965cdc8149f4acad42bb4f673ae2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=50), HTML(value='0.00% [0/50 00:00<00:00]'))), HTML(val…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: early stopping\n"
     ]
    }
   ],
   "source": [
    "learn.fit(50,1e-42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Problem: best result is not in the last epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=5), HTML(value='0.00% [0/5 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:19\n",
      "epoch  train loss  valid loss  accuracy\n",
      "0      0.849374    0.728057    0.555937  (00:04)\n",
      "1      0.844745    0.706033    0.575564  (00:03)\n",
      "2      0.843050    0.715870    0.562316  (00:03)\n",
      "3      0.861841    0.722247    0.558881  (00:04)\n",
      "4      0.853869    0.718867    0.555937  (00:03)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(5,1e-42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best epoch is #1. But model is in #4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveModel(LearnerCallback):\n",
    "    \"A `LearnerCallback` that terminates training when monitored quantity stops improving.\"\n",
    "    def __init__(self,\n",
    "                 monitor='val_loss',\n",
    "                 every = 'improvement',\n",
    "                 mode='auto'):\n",
    "        \n",
    "        super().__init__(learn)\n",
    "        \n",
    "        self.monitor = monitor\n",
    "        self.every = every\n",
    "        self.mode = mode\n",
    "        if every not in ['improvement', 'epoch']:\n",
    "            #should I use warning?\n",
    "            print(f'SaveModel every {every} is invalid, falling back to \"improvement\".')\n",
    "            every = 'improvement'\n",
    "        \n",
    "        if mode not in ['auto', 'min', 'max']:\n",
    "            #should I use warning?\n",
    "            print(f'SaveModel mode {mode} is invalid, falling back to \"auto\" mode.')\n",
    "            mode = 'auto'\n",
    "        if mode == 'min':\n",
    "            self.operator = np.less\n",
    "        elif mode == 'max':\n",
    "            self.operator = np.greater\n",
    "        else:\n",
    "            if 'loss' in self.monitor:\n",
    "                self.operator = np.less\n",
    "            else:\n",
    "                self.operator = np.greater\n",
    "   \n",
    "    def on_train_begin(self, **kwargs:Any)->None:\n",
    "        self.best = float('inf') if self.operator == np.less else -float('inf')\n",
    "    \n",
    "    def on_epoch_end(self, epoch, **kwargs:Any)->None:\n",
    "        if self.every==\"epoch\":\n",
    "            filename = f'model__epoch{epoch}'\n",
    "            learn.save(filename)\n",
    "            return \n",
    "        else: #every=\"improvement\"\n",
    "            current = self.get_monitor_value()\n",
    "            if current is None:\n",
    "                return\n",
    "            if self.operator(current, self.best):\n",
    "                self.best = current\n",
    "                filename = f'bestmodel_epoch{epoch}'\n",
    "                learn.save(filename)\n",
    "            return \n",
    "                                                                          \n",
    "    def get_monitor_value(self):\n",
    "        values = {'val_loss':learn.recorder.val_losses[-1:][0]\n",
    "               ,'trn_loss':learn.recorder.losses[-1:][0].cpu().numpy()}\n",
    "        for i, name in enumerate(learn.recorder.names[3:]):\n",
    "            values[name]=learn.recorder.metrics[-1:][0][i] \n",
    "\n",
    "        if values.get(self.monitor) is None:\n",
    "            print(f'Early stopping conditioned on metric `{self.monitor}` which is not available. Available metrics are: {\", \".join(map(str, learn.recorder.names[3:]))}')   \n",
    "        return values.get(self.monitor)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy], callbacks=[SaveModel(every='epoch')] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=5), HTML(value='0.00% [0/5 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:20\n",
      "epoch  train loss  valid loss  accuracy\n",
      "0      0.923145    0.757920    0.546124  (00:03)\n",
      "1      0.904519    0.760853    0.550540  (00:04)\n",
      "2      0.914042    0.753116    0.555447  (00:04)\n",
      "3      0.913810    0.758505    0.547596  (00:04)\n",
      "4      0.908953    0.764323    0.547105  (00:04)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(5,1e-42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model__epoch0.pth  model__epoch2.pth  model__epoch4.pth\r\n",
      "model__epoch1.pth  model__epoch3.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../data/mnist_sample/models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, tvm.resnet18, metrics=[accuracy], callbacks=[SaveModel(every='improvement')] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, max=5), HTML(value='0.00% [0/5 00:00<00:00]'))), HTML(value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:20\n",
      "epoch  train loss  valid loss  accuracy\n",
      "0      0.050595    0.040298    0.988714  (00:04)\n",
      "1      0.026141    0.011412    0.995093  (00:04)\n",
      "2      0.019063    0.012740    0.995093  (00:04)\n",
      "3      0.030391    0.009357    0.998037  (00:04)\n",
      "4      0.027890    0.007864    0.998528  (00:04)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(5,1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bestmodel_epoch0.pth  bestmodel_epoch4.pth  model__epoch2.pth\r\n",
      "bestmodel_epoch1.pth  model__epoch0.pth     model__epoch3.pth\r\n",
      "bestmodel_epoch3.pth  model__epoch1.pth     model__epoch4.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../data/mnist_sample/models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}