CustomCallbacks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CustomCallbacks.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "INTErZx5KHts"
      },
      "source": [
        "# Convolutional Neural Network\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGjbpVBoGjmu"
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Activation, Dense, Flatten, BatchNormalization, Conv2D, MaxPooling2D, Dropout\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IyUXJUueI8zY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76da0743-4d7a-4bf6-d70f-510d7c6c4c42"
      },
      "source": [
        "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n",
        "print(\"GPUs Available: \", len(physical_devices))\n",
        "tf.config.experimental.set_memory_growth(physical_devices[0], True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPUs Available:  1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3B5rd1c-Hti9"
      },
      "source": [
        "# Import MNIST Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HMT7OSHnHxhV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "70f5b240-a58a-4ff4-8347-e4a94060b7a4"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "print(x_train.shape)\n",
        "print(y_train.shape)\n",
        "print(y_train[:5])\n",
        "\n",
        "img_rows=x_train[0].shape[0]\n",
        "img_cols=x_test[0].shape[1]\n",
        "\n",
        "X_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
        "X_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "(60000, 28, 28)\n",
            "(60000,)\n",
            "[5 0 4 1 9]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wTzR3q-9IS8H"
      },
      "source": [
        "def plotImages(images_arr):\n",
        "    fig, axes = plt.subplots(1, 10, figsize=(20,20))\n",
        "    axes = axes.flatten()\n",
        "    for img, ax in zip( images_arr, axes):\n",
        "        ax.imshow(img)\n",
        "        ax.axis('off')\n",
        "    plt.tight_layout()\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12QjU55gIXyA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "b8926419-2eee-4ce2-d824-8a28d3ab7d79"
      },
      "source": [
        "plotImages(x_train[:10])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABZgAAACSCAYAAADIDq8FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5yV5Zk38GdmGKqAFEFQEHEYUSyooGIjKhqzr8FYkegbY6oVJVhWk002UbMajUbssYAlq67dmKiJBlk3Yu9KVUFEREVBpDnMzP7zlr3nOmYOj9P5fv+7frnOc+7I4Zzn3JzPfZXU1tZmAAAAAACwvkqbewEAAAAAALRONpgBAAAAAMjFBjMAAAAAALnYYAYAAAAAIBcbzAAAAAAA5GKDGQAAAACAXNr9o//xgNIja5tqIbQOf625q6SYPq8d6vLaIS+vHfLy2iEvrx3y8tohL68d8vLaIa9iXjteN9T1Za8bv2AGAAAAACAXG8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkIsNZgAAAAAAcrHBDAAAAABALjaYAQAAAADIxQYzAAAAAAC52GAGAAAAACAXG8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkIsNZgAAAAAAcrHBDAAAAABALjaYAQAAAADIxQYzAAAAAAC52GAGAAAAACCXds29ANjQrNtvl5AtPmltyF4ZdXNS7zjjuNDT/6r2ISub9uJXWB0AAHx1c6bEe953vn5jUl/6yeDQ89hRI0JW/eachlsYALQRvf7eI2SlJbUh+2iPZY2+Fr9gBgAAAAAgFxvMAAAAAADkYoMZAAAAAIBcbDADAAAAAJDLBj/kr6Rd/E9QtknvXNeafcagkFV3rknqLbb6MPR0PqkkZB9cGoe3vTjizqT+uHpl6Nntrkkhq/jJ0yGjadSM3ilkk2+6MmQV5fF1WFOnfmnUlNAze0R1yM4ctHvxC4T/YeURu4Xsot9ck9TnHfWd0FP7/OuNtiaa11sXjwrZzG/H97DykrKQ7XPSj5K60/3PNtzCgBavrFfPkJV07xaydw/vn9RresfBNBW/fCVkNatWfYXV0dDKhm0dsgf2vSpkVbXlSX1yj9mh5+4dDgxZ1ze/wuJo0Up2GRaymvbxu9Gir3VJ6jdOvTr0VNXG70YNaf/XjwhZl0MWJ3XNmjWNugb+sZIOHUK26hs7JvUOP42fKXNHrm20NUFDmnNjOgj3uYGXh55RT54cssHZy422pv/LL5gBAAAAAMjFBjMAAAAAALnYYAYAAAAAIJdWeQZz2TZDQlbbIT3P6/3RG4ee1bvHM4t7do/ZkzveGbKG8vCqriG76MqDQvbM9v8esneqVif1hUsOCD39n4zn1tF0qg5Mz8M56+pbQ09leTxfuyacuJxlb1dVJfXymnie1E4xytZ+Y2TIOk17LX2+DeRssNWH7BqzXulZsT1vmtFUy2nxPhwR/83xvPnfbIaV0Fw+mLhHUj8x7jehp6o2vocV5OMI2qzS7YYm9dxzOoWe723/VMgm9Xo01/Nt0/eEkA357gu5rkUjWfRBiCbMOTpkfx12T1OshhaidlR69u3c78Z7iMv2uz1k5SXrQjam04qkrqqN962FvlM1pL9u9x8hG37r95J6yxPfDz3VHy9ttDWRKjRPa9pV1yb1k2viNtjFW8bvPOveWdBwC4Mc5lwT9zOeO/CypF5RE790dZse78uagl8wAwAAAACQiw1mAAAAAAByscEMAAAAAEAuNpgBAAAAAMilxQ/5q/7aziG7dOpVISs0OK0lqKqtTuqfX/Hd0NNuZTyUe9Rdp4Ss66J02EGHj1eHns7PP7OeK6QYZd26hWzlPkNDNvGydDjjvp0+L3C14v5dZ+qn6bCtx68eFXr+/q+TQ/bXG64N2ba3pa+nwWdvGIPt3t8n/rfuvNWyNLipiRbT0pSWhah2YHxP2b/PrKR+vGSP0EPb8fmAdDhOz9KW+dnK+vvi6yNCtuCY9M/7xJ2nh57Te8wp6vrb33BqUndeHO9tlu2xNmRb/CG+T7d/9PminpOGVzJy+5DNmxg/L57Y68qk3qQsTh0uLXC/86dVPUL29to+SX1yj9mh59Z9rg/ZeSOPC1ntc6+FjKZRvWx5yBa8FwezZ8OaYDG0GLXnf5LUs4be20wraTwv75F+mfj6bieFng5/MuSvJdm7YxwiecHAniErNeSPZva1nWaGrGud72cnLTgo9PS+rnn2e/yCGQAAAACAXGwwAwAAAACQiw1mAAAAAAByscEMAAAAAEAuLX7IX4fZ74fshTUDQlZZvqTR1jBp8e4he/vz3iGbutXdIVtekw656Tv5qQZbVxyfQ2N575bNQvbcyDhssiH9qs9zSf3IRnG42vHzDwzZzYMeC1m3bTfMwRK/PPiukF00M/432xCVbbVFyGaNjhMPhz97bFL3N0Cpzfj8yN1Cds+hl9dJSkLPtcvigNPHjooD5LoseCOpa0IHjeWjE+JQ2CvOip9ZIzqkg4gLDWU7bv6YkO3U/d2QvfKDuq+dqND19+g5PmQ9H633UuRQtskmST3n8nhv88c9rg7Z4PLyAleLQ/3qmvJZvF+///C9QlbTIb3+yQ/FIX91X6tZlmWr+3YKWcd6V0VjKevbJ2R7b1PckFDarkVP1HkfiLcQBc1YE99jvvfnH6ZBvEUp+gvy7junr80pg/5S3ANpE8pK/M6S1OpDdg1Z70nvJPXacXHo8brFHzTYGj48Ke73XNT3spDd9ln6Pf7TcwaGntKsefZ//M0CAAAAACAXG8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkEuLH/JX6NDsKy46MmQXHLQyqcte3Sj0vHLSFUU95/kf75DU88Z0Dj3VyxaH7NujTgrZ/AlpvWX2SlFroPms22+XkN0+/MqQlWbt673W8Qv2D9nzj20Tste+H68/bXU6qqbP86tDz7xP46SM8l9PC1lpoSEYG4DyknXNvYQWq90Nq4rqW/1Wt0ZeCU1hzcFxcMUv/i0Odawsr//N4ubrDwrZpm823ABb/rGS8vSzZ82YHUPPPedcHLL+7eLApO8vOCCpF1yydejp8qeXQzatcxwmMv2+ynQNQx4MPYV89nKvkPUs6pGsr0XHDknqN0YXGsxYaKBf/W4rNNDvW3FYTfXsOPStZKdhuZ6TFqZrlxD9U8/nCjTW78Nd4mfRxq9Whqz6TUMEW7qBFz6f1If+RxzsWkjJF1UhG/LOMw2ypizLsmW908+ex57uGnrGdFpR1LX2e21cUneb9kboMei4ZamujX8iVZ3j1lj942xpK4698KGQHd9tYVKP2eXE0NPxoYYb8nfcyX8O2fAO8VX4w/MOTeqeT85osDV8VX7BDAAAAABALjaYAQAAAADIxQYzAAAAAAC5tPgzmAvpOSWeMbLJH9NzlKqXfhJ6hm33vZC9sU88g/LB349O6j7LijtbsmRGPF95y5ZzHApfomb0Tkk9+aZ4HnJFefyrUlPgNK2xs9LzcMqOWBl6Nv5ftSHb9tZTQlZ5VXrmT+nCl0JPjydDlFVdUB2ye3ZIX+ff23dC6Cmb9mK8WCtSs9fwkO3d8b+aYSWtw6AuS4vqG/BYfD3R+iw+dk3I9u0UsywrS6rj5o8JHZte7rzl5rT4lBFJ/ewZhc7Rjee1HTnvmyFbd3h6xmXnj+P5lvETK8ve/1GcVfDMkELrSD28Kp5xWXHdwpA5Pb9xbDZ2fq7H3f35piG7dE46Y6LvWfGVUj17blHX/3R7Z/23BdXz3gnZz/44LmSHj7+q3mu98e3JIdtp+WkhG+AM5havtuqLpK6ePa+ZVpJaclh6pvf27R8o0FXcCbzvv59ODtho1dt5l0Uz+nCXOINgwMPNsBCaxeIvNg5ZTbYgqdd1arjBVnX3oLIsyw7ZKM6Mq6rtFLJ1HVvugC2/YAYAAAAAIBcbzAAAAAAA5GKDGQAAAACAXGwwAwAAAACQS6sc8ldI9cf1D6yq+qx9UdcadsybSf3RNWWxqcbgq9aoZJdhIfv4J6uTurI8vk5eWBuv9bfPtw3Z0jsGJHWvT+OUx+63PR2zePkGHXLUtywdUrH09FWhp8+0BnzCZrDg4HgAfp+yzs2wkpan3aCBITui54NFPbbTO58mtXe+lq/d5puF7I29p4Ssqjb+ac5MZ75l715aGXq6ZHEQHI1j7hW7hWz2YekAkDhuNsu2+esJIRt6xvyQFXPvVMgJJxYahlS/8y84LmQ9FpqG3GR+mN4LbHvyqaFlwF/j+0KXNz4IWe8F6XC1r/LZsKpvyx1Ww1ez1Rnxnjcb3/TrYMP20YmjQjb02FlJXfe70vrY5qx0wKV75eZVW1UVsjlV6WDryvKOoWf1ll+EjLZp7uR4f31frzhg75pl6fegjZ9eFHqK3bMp2zjd8fn4jJWhp3+7+D408f09Qtb3xheSutBA7ubiF8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkIsNZgAAAAAAcmkzQ/6Ksc3Zc0J2/Pb7h2zKFo8n9egjTw49Xe8sMLSCFqW0cxzwtu43n4Xs6aH3JvU76+IB/z85d1LIejz5bsj6dPkwqVvqkIdd+y0I2fymX0aDalexoqi+NbM2buSVtDwLf9clZHt2iKPBbvxs8/jgZfHvDC1L2bCtk3rEv7+e+1rj7p2Q1Fvd47Ouqbz1291DNvuwq0K2vCYdVHPkrG+Hnq1Pjfc71Svqf48s7RLfK5YesUPIDtno4vjYLB20OvSueO9UMdVAv+ZUPS8dRFUx8Z0v6Uw15NDhQqpGFvf5TdtQXpIOT69qSdOJaFU+PCUOvzruxD+H7Nhul4Ssa2kc6l6M8z7aOWS1aw2Ha0mql3wYsglvjUvqR4bmG1ZM61O2dUXIbj34mpCtqo3DIe/96YFJ3Wnhs7nXMffqLZP69Z2vDz2Pre4aHzdybe7nbA5+wQwAAAAAQC42mAEAAAAAyMUGMwAAAAAAudhgBgAAAAAglw1qyF/1suUhW3riNiF798HVSf3P598Ses456tCQ1b7UPWQDLqgz0KbWJIumsnr0sJA9OvTqeh/3g9Mmhqzr/XHQVWMPvaFx9Hk+DrdrLcp69wrZksMrQ9bzqPeSenrljQWu1jEk11z1rZD1WfJU8QukWSwYm74u7u71UoGuspB8+61vhqzywreSuqUOKm3tyvr2CdnNh8bPp5osvl/VHerX/oA4tLXYd7nS4dsm9XY3zQw95/edXOCRHUKy58tHJ/XW/xqv5fXUdrz783S41rrOBe5vSwo8sEDbYUPqH/54yntfC1mnR14s5vK0MFW16TtBofc52o66g4jnHN8j9IzeK99w4ocGXBGywq+n+gf6zauK3+zGXROHvA+8b0l8zhVvhQxoHrV7Dk/qo298KPSM6BDvSIc+clrIKu/PN9Rv/vmjQvb8PpfWSeJW7Nk3fC9km2Wt67u4XzADAAAAAJCLDWYAAAAAAHKxwQwAAAAAQC4b1BnMhdS8Es8IPPqXZyb1H35xSeh5efd4LnO2e4yGdTklqYdcvzj0rHt7/j9eJLnscN7LISst8G8qxy/YP6k75Txrp6UoL4lnrVbVOZSwrGTDPaVwdc/0NdDlK1yrZu+dkrq2LB44uXBMPKv0i/5VISttn54F9Ze947ly5QXOs/ygOl7/X95Oz4j/pCaeR9e5NJ491feZFSHbcF8pLdMnx8czve474eI6SXnoOWHh6JBVHRdfO9UfvZt7bRSvpGP8b1/oPLhCOk1Iz5Is2WJA6Jl7wuYhO3BMPK92Yp/fJ/XAdp1CT6HTLKsLzJMoubN32rNsboFH0pKUdesWsjW7DglZ+TnxzNFXh8bPqPC4gvcj9b/Op63uHLL3fjQwZLXr4j080Hzqnn2aZVn23Sn3JfUhXT5uwGdsuN/KTZg3LmSbXRTPPjVLoO3aqOeq5l4C/0BJeTxLffEpI0L2/Bnp/Unhe5H43nHY8Hif/OBF6feuil++EnpKN41zVcb+U5zfVVZnMMXwp+J5ywMvbF3nLRfiF8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkIsNZgAAAAAActngh/wV0vOmGUl9yuyTQ0+3C98L2e2DHw3ZG9+5MqmHDvhB6Nn6l3Gfv3ru2/Wuk/9v2f+Og69+1jcOZ6zJ4uHwL/xl26QemLXuw9ULDdCpqTOm6ZGZ24aeIVk82L41WbsmDjarKTCibsq5lyX1g6fEgSTFOrvXDUldmsUpfKtrvwjZ+9Xxz+jKj76W1GMeOz30bPxSfP32+0scvlSyIH1/+mhmHNzVtywOGqx97rWQ0XzKhm0dsqfOv7JAZ8d6rzXjvUEhGzD/9RyroiHUrlkbsmfWxvew3TrEv6cPPHZHUtd9f18fj61OB/PNrTsRNsuyfTt9HrLnv4jvRRvfMiNkNJ+SDgUGzI7ePqknXn1r6Nm30+MhW1IdX6/TVvdI6p/POST03D5sasj6t4vrqqtjaXzdv33UxiEbPDu+99WsWVPv9YGmU1bnXrzQwPW8ihlsXqxHtrkvZHsfE/cAuv8hDu+ibbhn5+tDdmq2ZzOshEI+OCEO9Hv2jMtDVveuuNB7wi2fbRayX2/6TMyOTbNzx+wWeg7o/nDICt07P7M2vWcZeGTb/N7tF8wAAAAAAORigxkAAAAAgFxsMAMAAAAAkIsNZgAAAAAAcjHkrwglf385ZKuO6BOykeNODdkzZ6cHj8/a94bQc8ygA0O2fK/1WSHr4gyzrHtpHEI0Y00cLjP4lvfTazXYqhpWaefOIZt1yXYFOl8IyTFvfyOph572TuiJY+dal4pjXwrZsH87JWQDRi5qsOec9mFlUn/08Oahp9cbcVhR+0eeK3C1tK8ye76oNRT6c1t09h5JPbJDHL51x+dxuAEty5xz49/5QkM8izHwwpjlnINDA6he8mHIfnFiHAJ8ybVXh2yHOh9tt302IPScP31syCqnxuFn7ZYsT+o+t38SevYd8LeQHTctrrXY9ywaXmnHOOxu6bidQvbkryfXe61ht8d72c2nxfedDn9KP8d69YsDbW5/dJeQTepV/3DRQsMtX/1uXPuohRNC1veWV5K6ZtWqep+PplV3MFuxQ9m67RHfN2lZCn1nvvFbByX1P3+3V+gZ+GgciF22uuG+kc39fhyiO+ugaxrs+rR8C/+rzr3S0OZZB8X56IRRIXvq7N+FbEVNvF94s6pLUv/0jB+Hno5L43vO47+eH7Ipg/6S1IUGARYaXFpo/PaI9ulzTpw3M/Rcfvhh8VqvxL6WzC+YAQAAAADIxQYzAAAAAAC52GAGAAAAACAXG8wAAAAAAORiyF9OhQb09J0cszVnpQMKOpfEwXPXD3ooZAcfenrIOt8XDxVn/Syt3ihk696e3/QLKULdoX6zL9w+9Mw65MqQPbyqe8jev6oiqbt++vRXXF3rsOU5cbhdY+qXvdukz/dlOu/zUb09P5t2eMgqs2cbYzkUoWZ0HMh1/oj7c13rgNePDtlGz9c/WIvm1f7ROCTv3C13zXWtYv8urzgkvf6fBj4Qeqpq428ROs2P9zI0jZIOcVjxrEt3iNkh9Q/0O2T2t0JWefHbISt0z9tuQDrUdscH4+ffmb3eDNnymjhYZ7d7JiV1v6Hx+R7f/s6QzfiX+P9x3PiDk/rjyfHeqePSOBSokLInXiyqj/VTd1htTcFxSNH0HW8P2djdv58GT7+ae100juo35yT14LOafg3bzN0khgfFiLZro4X1TxPtWhJ7yratDFnd1zQNb9vvxMF2D67sG7Jf/358yPr99qmk7pwVt4e2dFK8l5p4xd5JfVn/J4u6ViFlJSVJfeZr8bt4/1fifVNr4xfMAAAAAADkYoMZAAAAAIBcbDADAAAAAJCLM5iLULPX8JC9dWTHkG03fH7ICp25XNcVn8SzNzs/EM9i5Ks74+9Hhqwye6EZVpIqdP7qhz9ZndQzR8Tzlvd/bVzIuhwUz0/smm0YZy5TvC0eqP8sMprOBVN/H7Ltyov7Mzpj8T5J3X38p6GnOiSQZes6pb8zqHs2apYVPh91y6nxvN11IaEhlLRLb9Vn/27H0DNr7FUhe2/d2pCNvS49/HTQTW+FnnUFzluuGrNLyLa76KWk/kWfeC815bMtQnbrT78Zsop703uUst69Qs/XDjg1ZCvHLQ/ZfTtdn9SbT45nVhfy0Mr4nL+vHFzUY1k/Q//2g6R+c7/4+VesOT9Kv2dVut2lgCWHVdTfRJtWWsRNSt0zcrMsy2o6lTfCaqjPC49uG7JP7ugdsn6znwpZXqv7xv29Uzf5W50kvh52/9UpIev9ysp6n2/AvEUhawvf1/yCGQAAAACAXGwwAwAAAACQiw1mAAAAAAByscEMAAAAAEAuG/yQv5IR24VszoR0YMT1e94cevbp+EWu51tbWxWypz/ZMjbWLM51/Q1WPJM/Ky3w7yeX73V7yK7KKhtjRV9qwa9Gheye71wassry9HW487PHhZ7+h77ZcAsDms1O7eP7VaGBa4XMmLJzUvf5tOEGXtC2db2jzkSs3zbPOvhyC8/cNalnjb089LxfYKDfkReeGbJB96dDgD/ZL95/1h7bNWR3bxefc5OydHjesDviEL7K338css6znwlZXdUfLw1Zt9sLZfGxR5yUDjLse8SCep8vy7Ism7RxgfCN4h7Leukwp1Ma7Nc866B4JR3isMxlR8YB5T0eiH9nalasaJQ1fZnFk/YI2QMTflOgs7gBoLQNPabOSOprz4pDaE/oHj8v5k5sH7KKYxtuXRQ28Jfxu0xDDsAr22STkL13eJwEWVGevk/8YUW/0NP7uhkhK0ZbGOhXiF8wAwAAAACQiw1mAAAAAAByscEMAAAAAEAuNpgBAAAAAMilzQ75a7dlPLj9reP7h+xfx90RssM3ikNJ8jp3yYiknn757qGnx835Dgbnf6iNUU1WE7LRneKQmNOn7pLUW02Jjyv/IA6oWDI6Hg7fc9x7SX3qwMdDzzc6vxCyB1f2Ddl3XjsoqXtf1yX0QDHKSuK/JX5aWR6yTR9uitWQZVm28O50wGx5ycu5r9XvifQzq60OjaDhrTi67j1J/HyieV3zw6vr7elYYNDxN0/4z5BtNuHTpD6u2x+LXEUchjXs3yckdcU5z4We6nVxYE5j63N1Ohiotv7/fP/HogZfC4UNOC/9M7r9mM1CzzFdixt2/s5BNyT1N3YcH3pqXpm5Hqsjy7JszTfT4aLdz3g39EyvuCJkhz4X//tnsxtuyF+7fpsm9aIjBoeeO0+9JGT929U/0G9JdRyWWr66wBdM2oRLnv56yA7a/3chq/zxnJDFnQJam7mTKkI2c//JIZuxNv2+/B9j9y5wtbcaalltgl8wAwAAAACQiw1mAAAAAAByscEMAAAAAEAuNpgBAAAAAMilVQ75azdoYMiW79Ivqcf96pHQc8LG9zbYGiYtjsP6Zlw9ImQ9pz6b1D1qDPRrTh1L4kt+5gHXJvV/7d0x9Mxdu2nIju8+P9caTns/Hg7/yFPDQzbktKdzXR/qqq4tMI7CPy82mZrRO4Xsd8NvS+qq2jiab3nNmpCNfPj0kA1d8OZXWB0bsuWDvRG0dP/5+dCk3q3Da6GnZ1kcYHVu7/oHhx4867CQvTtj85ANvnt5yCreSAdC1jbDQD/ahqnv7hGy8cPuKuqxVWawNYqvXzA9qSf1er2ox806t1sMP9+tIZaUZVmWHb1H+j36/j5/Cj01WRxiXchx89Mhb/OmbB16et3re/uGpDqLE3NrVsd7cVqXsm0rQ3beoXeErLo2fqAc/+AJSV0xx/5MfXyzAAAAAAAgFxvMAAAAAADkYoMZAAAAAIBcWtQZzO36xXNuP7mpS8hO3HJ6yMZ3XdJg6zhl0V5J/eI18Xzc3nfHs6h6rnBOU3Pp+8SHITv7x6NCdtGm9f8Z7dPxi5Dt1XF+Uet4aW36bzbjp/8o9FQe/0LIhmTO86FprRq5qrmXsMFY07N9yPbquLJOUhZ6Hl0V5w1U/ui5kBU4YRuKstn09H2g/JT4OnTGafN6at/+Sb3bMfuFnuU7xvuWdh/Fc0grr12U9nwQ750GrVkYMu8xNKa1U+P3v+zipl8HX93MMdc18TPG38rNWBPPpP/hM98JWcUP5yZ1r5W+x2/otmrXKWRLj981ZL1u9FppTY6694mQHbpRvP/Z+enjQ1Zxuj2a9eUXzAAAAAAA5GKDGQAAAACAXGwwAwAAAACQiw1mAAAAAAByabIhf198fUTMJn6S1OdW/Dn0HNip7iCk/JZUrw7ZPg9OCtnQn81K6p7L4kHuBp60LNVz3grZ3CMHhWzbU08N2ZtHXZHrOYf++aSQbX11OjCp8qU40A+aWlmJf0sEopK/v5zUUz/rE3rGd10UslXD+oWs/cL3Gm5h/D/VS9N75b6Tnwo9fYu81roGWA80tB4vfxKyqz7dOmQn95jdFMshy7K/TdgzqW85KQ46e2XPmxp1Dbd9NiBki6s2TuqbXtwz9FRcXx2ywXU+67LMd/kN3ZTR8fX7aU3cK+r96uchM/u4dbnggcNDNv7YySHr9OduTbGcNs+uAwAAAAAAudhgBgAAAAAgFxvMAAAAAADkYoMZAAAAAIBcmmzI3/xvxb3sOdvfletaVy3bKmSXTz8wqUuqS0LP0PPfCdmQJc+ELI4GoDVa9/b8kFVMjNnYiSNzXb8yey5kDv2nJVj72CZJXT3cKJPm1O3lD0J26nv7JfW1A6Y31XLgS1123REhG3/G5SHr9y/zQrZ02Q5p8PSrDbYuoO2qfnNOyB7dLg5bejQr5n59ZgOsiLInXkzqLZ/tHHp2mXBayG7+8e9Ctl379Dv5fq+NCz3Ln9g0ZFvcGQfMrntnQVIPyQxTJ58zZ8b7nSO2eClkpSvXhsxeUesy+OwZIRt7dvw86ZXFPtafXzADAAAAAJCLDWYAAAAAAHKxwQwAAAAAQC42mAEAAAAAyKXJhvxVnvhsyA4+cZeGu34Wr1+XA9mBDcGmlz2V1P902c6hZ3D2clMtZ4NXdyhNlmXZe7un9cFZw30eQl6b3To7ZOO+dXDI7qx4KGSjfz4+qXt+u3voqV62/CusDoDmULNqVcg2u/CpkJ174dUR4VoAAAEISURBVK71Xmuj7O2isnVFrg3y6HlwHC76t6xLgc7YB3w5v2AGAAAAACAXG8wAAAAAAORigxkAAAAAgFya7AxmAABaruqPl4bsi8N7hWyb3/44ZDPHXJfUY4d+Pz7B06/mXxwAANBi+QUzAAAAAAC52GAGAAAAACAXG8wAAAAAAORigxkAAAAAgFwM+QMAoKBCg/+GHBezsdnIOomBfgAAsKHwC2YAAAAAAHKxwQwAAAAAQC42mAEAAAAAyMUGMwAAAAAAuZTU1tY29xoAAAAAAGiF/IIZAAAAAIBcbDADAAAAAJCLDWYAAAAAAHKxwQwAAAAAQC42mAEAAAAAyMUGMwAAAAAAufw31oSRubJFgaQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1440x1440 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K5STVm1bJ_xc"
      },
      "source": [
        "# Create a Convolutional Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QfW99cTqI28u"
      },
      "source": [
        "model = Sequential([\n",
        "      Conv2D(32, (5,5), activation='relu', padding='same', input_shape=(28, 28, 1)),\n",
        "      MaxPooling2D(pool_size=(2,2), padding='same'),\n",
        "      Conv2D(32, (5,5), padding='same', activation='relu'),\n",
        "      MaxPooling2D(pool_size=(2,2), padding='same'),\n",
        "      Flatten(),\n",
        "      Dense(1024, activation='relu'),\n",
        "      Dropout(0.4),\n",
        "      Dense(10)\n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bnZtiLX-J-eY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a039e37a-f02b-4466-a559-2ba5c9e28bd1"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 28, 28, 32)        832       \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 14, 14, 32)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 14, 14, 32)        25632     \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 7, 7, 32)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1568)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1024)              1606656   \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                10250     \n",
            "=================================================================\n",
            "Total params: 1,643,370\n",
            "Trainable params: 1,643,370\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GODLG0GJIKBX"
      },
      "source": [
        "# Compile the Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qwaXDxfzINSl"
      },
      "source": [
        "model.compile(optimizer=Adam(learning_rate=0.001),\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['sparse_categorical_accuracy']\n",
        "              )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "avQLqMzaI3Dx"
      },
      "source": [
        "# Train the Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1GOHli9QuAmY"
      },
      "source": [
        "Built-in Callback Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJOrJ22xtiz1"
      },
      "source": [
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n",
        "\n",
        "# Checkpoint during training for best model\n",
        "checkpoint = ModelCheckpoint(\"/content/gdrive/MyDrive/ColabNotebooks/best_model.h5\", monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n",
        "\n",
        "earlyStopping = EarlyStopping(\n",
        "    monitor=\"val_loss\",\n",
        "    patience=5,\n",
        "    restore_best_weights=True\n",
        ")\n",
        "\n",
        "reduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n",
        "                              patience=3, min_lr=0.00001)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QSOicuAEt806"
      },
      "source": [
        "### Create Your Own Callback Function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sXpVWAPlkmSn"
      },
      "source": [
        "Basic Callback Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFB_XsXcuOCI"
      },
      "source": [
        "class CustomCallback(keras.callbacks.Callback):\n",
        "    def on_train_begin(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Starting training; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_train_end(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Stop training; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Start epoch {} of training; got log keys: {}\".format(epoch, keys))\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"End epoch {} of training; got log keys: {}\".format(epoch, keys))\n",
        "\n",
        "    def on_test_begin(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Start testing; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_test_end(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Stop testing; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_predict_begin(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Start predicting; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_predict_end(self, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"Stop predicting; got log keys: {}\".format(keys))\n",
        "\n",
        "    def on_train_batch_begin(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Training: start of batch {}; got log keys: {}\".format(batch, keys))\n",
        "\n",
        "    def on_train_batch_end(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Training: end of batch {}; got log keys: {}\".format(batch, keys))\n",
        "\n",
        "    def on_test_batch_begin(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Evaluating: start of batch {}; got log keys: {}\".format(batch, keys))\n",
        "\n",
        "    def on_test_batch_end(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Evaluating: end of batch {}; got log keys: {}\".format(batch, keys))\n",
        "\n",
        "    def on_predict_batch_begin(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Predicting: start of batch {}; got log keys: {}\".format(batch, keys))\n",
        "\n",
        "    def on_predict_batch_end(self, batch, logs=None):\n",
        "        keys = list(logs.keys())\n",
        "        print(\"...Predicting: end of batch {}; got log keys: {}\".format(batch, keys))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Ur6d7yukljX"
      },
      "source": [
        "Using the Logs in Callback Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SjDZA782kkmD"
      },
      "source": [
        "class LossAndErrorPrintingCallback(keras.callbacks.Callback):\n",
        "    def on_train_batch_end(self, batch, logs=None):\n",
        "        print(\"For batch {}, loss is {:7.2f}.\".format(batch, logs[\"loss\"]))\n",
        "\n",
        "    def on_test_batch_end(self, batch, logs=None):\n",
        "        print(\"For batch {}, loss is {:7.2f}.\".format(batch, logs[\"loss\"]))\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        print(\"The average loss for epoch {} is {:7.2f} \".format(\n",
        "                epoch, logs[\"loss\"]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hWEbqBl8k7sJ"
      },
      "source": [
        "Create Your Own Learning Rate Scheduler"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S71aPdz2k-o_"
      },
      "source": [
        "class CustomLearningRateScheduler(keras.callbacks.Callback):\n",
        "\n",
        "    def __init__(self, schedule):\n",
        "        super(CustomLearningRateScheduler, self).__init__()\n",
        "        self.schedule = schedule\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs=None):\n",
        "        if not hasattr(self.model.optimizer, \"lr\"):\n",
        "            raise ValueError('Optimizer must have a \"lr\" attribute.')\n",
        "        # Get the current learning rate from model's optimizer.\n",
        "        lr = float(tf.keras.backend.get_value(self.model.optimizer.learning_rate))\n",
        "        # Call schedule function to get the scheduled learning rate.\n",
        "        scheduled_lr = self.schedule(epoch, lr)\n",
        "        # Set the value back to the optimizer before this epoch starts\n",
        "        tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)\n",
        "        print(\"\\nEpoch %05d: Learning rate is %6.4f.\" % (epoch, scheduled_lr))\n",
        "\n",
        "\n",
        "LR_SCHEDULE = [\n",
        "    # (epoch to start, learning rate) tuples\n",
        "    (1, 0.05),\n",
        "    (2, 0.01),\n",
        "    (3, 0.005),\n",
        "    (4, 0.001),\n",
        "]\n",
        "\n",
        "\n",
        "def lr_schedule(epoch, lr):\n",
        "    \"\"\"Helper function to retrieve the scheduled learning rate based on epoch.\"\"\"\n",
        "    if epoch < LR_SCHEDULE[0][0] or epoch > LR_SCHEDULE[-1][0]:\n",
        "        return lr\n",
        "    for i in range(len(LR_SCHEDULE)):\n",
        "        if epoch == LR_SCHEDULE[i][0]:\n",
        "            return LR_SCHEDULE[i][1]\n",
        "    return lr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABjOAS552RAo"
      },
      "source": [
        "## Train Neural Network with Callback Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_GXN5MJJY6A"
      },
      "source": [
        "batch_size = 200\n",
        "epochs = 3\n",
        "steps_per_epoch = 54000 // batch_size\n",
        "validation_steps = 6000 // batch_size"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qfcdM20IJABW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c995f8ef-2ca7-48f5-b723-f9f054741cc9"
      },
      "source": [
        "history = model.fit(X_train, y_train,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    steps_per_epoch=steps_per_epoch,\n",
        "                    validation_split=0.1, \n",
        "                    validation_steps=validation_steps,\n",
        "                    #callbacks=[CustomCallback()]\n",
        "                    #callbacks=[LossAndErrorPrintingCallback()]\n",
        "                    callbacks=[CustomLearningRateScheduler(lr_schedule)]\n",
        "                    )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "\n",
            "Epoch 00000: Learning rate is 0.0010.\n",
            "  6/270 [..............................] - ETA: 2s - loss: 0.0143 - sparse_categorical_accuracy: 0.9975WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0030s vs `on_train_batch_end` time: 0.0072s). Check your callbacks.\n",
            "270/270 [==============================] - 2s 8ms/step - loss: 0.0213 - sparse_categorical_accuracy: 0.9935 - val_loss: 0.0815 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 2/5\n",
            "\n",
            "Epoch 00001: Learning rate is 0.0500.\n",
            "270/270 [==============================] - 2s 8ms/step - loss: 399.0356 - sparse_categorical_accuracy: 0.2360 - val_loss: 1.8886 - val_sparse_categorical_accuracy: 0.3532\n",
            "Epoch 3/5\n",
            "\n",
            "Epoch 00002: Learning rate is 0.0100.\n",
            "270/270 [==============================] - 2s 8ms/step - loss: 1.9724 - sparse_categorical_accuracy: 0.3021 - val_loss: 1.8240 - val_sparse_categorical_accuracy: 0.3547\n",
            "Epoch 4/5\n",
            "\n",
            "Epoch 00003: Learning rate is 0.0050.\n",
            "270/270 [==============================] - 2s 8ms/step - loss: 1.9437 - sparse_categorical_accuracy: 0.3130 - val_loss: 1.8211 - val_sparse_categorical_accuracy: 0.3632\n",
            "Epoch 5/5\n",
            "\n",
            "Epoch 00004: Learning rate is 0.0010.\n",
            "270/270 [==============================] - 2s 8ms/step - loss: 1.9573 - sparse_categorical_accuracy: 0.3146 - val_loss: 1.8255 - val_sparse_categorical_accuracy: 0.3627\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4DDvShiL8ME"
      },
      "source": [
        "# Plot the accuracy of the model during training\n",
        "plt.plot(history.history['sparse_categorical_accuracy'])\n",
        "plt.plot(history.history['val_sparse_categorical_accuracy'])\n",
        "plt.title('Model Accuracy')\n",
        "plt.ylabel('Sparse_categorical_accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Training', 'Validation'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "esRRCfC2L9gb"
      },
      "source": [
        "# Plot the loss of the model during training\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model Loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Training', 'Validation'], loc='upper right')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ipteh02QObYC"
      },
      "source": [
        "# Evaluate the best model with testing data.\n",
        "print(model.evaluate(X_test, y_test))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VJePt7poPoPa"
      },
      "source": [
        "plotImages(x_test[:10])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xStbKUBhOxu6"
      },
      "source": [
        "predictions = np.argmax(model.predict(X_test[:10]), axis=-1)\n",
        "print(predictions)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}