Deep Learning/05. Tensorboard, Fine-Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorboard for Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "from pathlib import Path\n",
    "\n",
    "# Reset the name counter\n",
    "def cls():\n",
    "    tf.keras.backend.clear_session()\n",
    "    tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (3870, 8) (5160, 8)\n"
     ]
    }
   ],
   "source": [
    "housing = datasets.fetch_california_housing()\n",
    "x_train, x_test, y_train, y_test   = train_test_split(housing.data, housing.target, random_state=42)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, random_state=42)\n",
    "print(x_train.shape, x_valid.shape, x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls()\n",
    "norm_layer = keras.layers.Normalization(input_shape=x_train.shape[1:])\n",
    "model = keras.Sequential([\n",
    "    norm_layer, keras.layers.Dense(30, activation=\"relu\"),\n",
    "    keras.layers.Dense(30, activation=\"relu\"),\n",
    "    keras.layers.Dense(1)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3), metrics=[\"RootMeanSquaredError\"])\n",
    "norm_layer.adapt(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "normalization (Normalization (None, 8)                 17        \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,248\n",
      "Trainable params: 1,231\n",
      "Non-trainable params: 17\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.8866 - root_mean_squared_error: 1.3736 - val_loss: 0.7126 - val_root_mean_squared_error: 0.8442\n",
      "Epoch 2/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.6577 - root_mean_squared_error: 0.8110 - val_loss: 0.6880 - val_root_mean_squared_error: 0.8295\n",
      "Epoch 3/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5934 - root_mean_squared_error: 0.7703 - val_loss: 0.5803 - val_root_mean_squared_error: 0.7618\n",
      "Epoch 4/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5557 - root_mean_squared_error: 0.7455 - val_loss: 0.5166 - val_root_mean_squared_error: 0.7188\n",
      "Epoch 5/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5272 - root_mean_squared_error: 0.7261 - val_loss: 0.4895 - val_root_mean_squared_error: 0.6997\n",
      "Epoch 6/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5033 - root_mean_squared_error: 0.7094 - val_loss: 0.4951 - val_root_mean_squared_error: 0.7036\n",
      "Epoch 7/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4854 - root_mean_squared_error: 0.6967 - val_loss: 0.4862 - val_root_mean_squared_error: 0.6973\n",
      "Epoch 8/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.4709 - root_mean_squared_error: 0.6862 - val_loss: 0.4554 - val_root_mean_squared_error: 0.6748\n",
      "Epoch 9/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.4578 - root_mean_squared_error: 0.6766 - val_loss: 0.4413 - val_root_mean_squared_error: 0.6643\n",
      "Epoch 10/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4474 - root_mean_squared_error: 0.6689 - val_loss: 0.4379 - val_root_mean_squared_error: 0.6617\n"
     ]
    }
   ],
   "source": [
    "tensorboard_path = f'tensorboard/{datetime.now().strftime(\"_%Y_%m_%d_%H_%M_%S\")}'\n",
    "cb_tensorboard = keras.callbacks.TensorBoard(tensorboard_path)\n",
    "history = model.fit(x_train, y_train, validation_data=(x_valid, y_valid), epochs=10,callbacks=[cb_tensorboard])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  _2022_06_02_20_11_27\n",
      "    train\n",
      "      events.out.tfevents.1654180887.RPL-RAJ-105.22428.0.v2\n",
      "      events.out.tfevents.1654180887.RPL-RAJ-105.profile-empty\n",
      "      plugins\n",
      "        profile\n",
      "          2022_06_02_14_41_27\n",
      "            RPL-RAJ-105.input_pipeline.pb\n",
      "            RPL-RAJ-105.kernel_stats.pb\n",
      "            RPL-RAJ-105.memory_profile.json.gz\n",
      "            RPL-RAJ-105.overview_page.pb\n",
      "            RPL-RAJ-105.tensorflow_stats.pb\n",
      "            RPL-RAJ-105.trace.json.gz\n",
      "            RPL-RAJ-105.xplane.pb\n",
      "    validation\n",
      "      events.out.tfevents.1654180888.RPL-RAJ-105.22428.1.v2\n",
      "  _2022_06_02_20_12_19\n",
      "    train\n",
      "      events.out.tfevents.1654180940.RPL-RAJ-105.22428.2.v2\n",
      "      events.out.tfevents.1654180940.RPL-RAJ-105.profile-empty\n",
      "      plugins\n"
     ]
    }
   ],
   "source": [
    "for idx, path in enumerate(sorted(Path(\"tensorboard\").glob(\"**/*\"))):\n",
    "    if idx>20:\n",
    "        break\n",
    "    print(\"  \" * (len(path.parts) - 1) + path.parts[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-c21fe22d4fd95112\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-c21fe22d4fd95112\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "%tensorboard --logdir=./tensorboard "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://localhost:6006/  <-- Go to this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Till now, we just wrote some scaler values over there. You can do more things as well.\n",
    "\n",
    "# To visualize histogram, to listen audio, image, text\n",
    "import numpy as np\n",
    "tensorboard_path = f'tensorboard/{datetime.now().strftime(\"_%Y_%m_%d_%H_%M_%S\")}'\n",
    "writer = tf.summary.create_file_writer(str(tensorboard_path))\n",
    "with writer.as_default():\n",
    "    for step in range(1, 1000 + 1):\n",
    "        tf.summary.scalar(\"my_scalar\", np.sin(step / 10), step=step)\n",
    "        \n",
    "        data = (np.random.randn(100) + 2) * step / 100  # gets larger\n",
    "        tf.summary.histogram(\"my_hist\", data, buckets=50, step=step)\n",
    "        \n",
    "        images = np.random.rand(2, 32, 32, 3) * step / 1000  # gets brighter\n",
    "        tf.summary.image(\"my_images\", images, step=step)\n",
    "        \n",
    "        texts = [\"The step is \" + str(step), \"Its square is \" + str(step ** 2)]\n",
    "        tf.summary.text(\"my_text\", texts, step=step)\n",
    "        \n",
    "        sine_wave = tf.math.sin(tf.range(12000) / 48000 * 2 * np.pi * step)\n",
    "        audio = tf.reshape(tf.cast(sine_wave, tf.float32), [1, -1, 1])\n",
    "        tf.summary.audio(\"my_audio\", audio, sample_rate=48000, step=step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can share your TensorBoard logs with the world by uploading them to https://tensorboard.dev/. For this, you can run the tensorboard dev upload command, with the --logdir and --one_shot options, and optionally the --name and --description options. The first time, it will ask you to accept Google's Terms of Service, and to authenticate. This requires user input. Colab supports user input from shell commands, but the main other Jupyter environments do not, so for them we use a hackish workaround (alternatively, you could run the command in a terminal window, after you make sure to activate this project's conda environment and move to this notebook's directory)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboard.main import run_main\n",
    "import sys\n",
    "\n",
    "argv = \"tensorboard dev upload --logdir ./tensorboard --one_shot\".split()\n",
    "argv += [\"--name\", \"Quick test\", \"--description\", \"This is a test\"]\n",
    "try:\n",
    "    original_sys_argv_and_sys_exit = sys.argv, sys.exit\n",
    "    sys.argv, sys.exit = argv, lambda status: None\n",
    "    run_main()\n",
    "finally:\n",
    "    sys.argv, sys.exit = original_sys_argv_and_sys_exit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!tensorboard dev list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To list down all active states\n",
    "from tensorboard import notebook\n",
    "notebook.list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine Tuning the Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Keras Tuner has four tuners available - RandomSearch, Hyperband, BayesianOptimization, and Sklearn."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01. Using Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n",
    "\n",
    "def build_model(hp):\n",
    "    n_hidden      = hp.Int(\"n_hidden\", min_value=0, max_value=8, default=2)\n",
    "    n_neurons     = hp.Int(\"n_neurons\", min_value=16, max_value=256)\n",
    "    learning_rate = hp.Float(\"learning_rate\", min_value=1e-4, max_value=1e-2,sampling=\"log\")\n",
    "    optimizer     = hp.Choice(\"optimizer\", values=[\"sgd\", \"adam\"])\n",
    "    if optimizer == \"sgd\":\n",
    "        optimizer = keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "    else:\n",
    "        optimizer = keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "    model = keras.Sequential()\n",
    "    model.add(keras.layers.Flatten())\n",
    "    for _ in range(n_hidden):\n",
    "        model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
    "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 3 Complete [00h 00m 03s]\n",
      "val_accuracy: 0.004392764996737242\n",
      "\n",
      "Best val_accuracy So Far: 0.004651162773370743\n",
      "Total elapsed time: 00h 00m 12s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0603 08:38:13.061116 16832 <ipython-input-25-6130168ecc14>:3] Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "random_search_tuner = kt.RandomSearch(build_model, objective=\"val_accuracy\", max_trials=3, overwrite=True,\n",
    "    directory=\"california_house\", project_name=\"my_rnd_search\", seed=42)\n",
    "random_search_tuner.search(x_train, y_train, epochs=4, validation_data=(x_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.engine.sequential.Sequential at 0x2049a3db190>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top3_models = random_search_tuner.get_best_models(num_models=3)\n",
    "best_model = top3_models[0]\n",
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_hidden': 5,\n",
       " 'n_neurons': 25,\n",
       " 'learning_rate': 0.0006562536901904111,\n",
       " 'optimizer': 'adam'}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top3_params = random_search_tuner.get_best_hyperparameters(num_trials=3)\n",
    "top3_params[0].values  # best hyperparameter values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial summary\n",
      "Hyperparameters:\n",
      "n_hidden: 5\n",
      "n_neurons: 25\n",
      "learning_rate: 0.0006562536901904111\n",
      "optimizer: adam\n",
      "Score: 0.004651162773370743\n"
     ]
    }
   ],
   "source": [
    "best_trial = random_search_tuner.oracle.get_best_trials(num_trials=1)[0]\n",
    "best_trial.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.004651162773370743"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_trial.metrics.get_last_value(\"val_accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.5108 - accuracy: 0.0028\n",
      "Epoch 2/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.4100 - accuracy: 0.0029\n",
      "Epoch 3/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.3618 - accuracy: 0.0028\n",
      "Epoch 4/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.3570 - accuracy: 0.0019\n",
      "Epoch 5/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.2990 - accuracy: 0.0021\n",
      "Epoch 6/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.2474 - accuracy: 0.0023\n",
      "Epoch 7/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.2650 - accuracy: 0.0023\n",
      "Epoch 8/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.2166 - accuracy: 0.0021\n",
      "Epoch 9/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.1978 - accuracy: 0.0017\n",
      "Epoch 10/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.2046 - accuracy: 0.0022\n",
      "162/162 [==============================] - 0s 1ms/step - loss: 1.1402 - accuracy: 0.0016\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.1401960849761963, 0.001550387591123581)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.fit(x_train, y_train, epochs=10)\n",
    "test_loss, test_accuracy = best_model.evaluate(x_test, y_test)\n",
    "test_loss, test_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 02. Using Keras Wrapper of scikit learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(n_hidden=1, n_neurons=10, learning_rate=1e-3, input_shape=[8]):\n",
    "    model = keras.models.Sequential()\n",
    "    model.add(keras.layers.InputLayer(input_shape=input_shape))\n",
    "    for _ in range(n_hidden):\n",
    "        model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(1))\n",
    "    optimizer = keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "    model.compile(loss=\"mse\", optimizer=optimizer)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1909799413469987995648.0000 - val_loss: 541821669262491648.0000\n",
      "Epoch 2/4\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 286294118995329024.0000 - val_loss: 126656876162056192.0000\n",
      "Epoch 3/4\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 66924463874113536.0000 - val_loss: 29607445101805568.0000\n",
      "Epoch 4/4\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 15644358246137856.0000 - val_loss: 6921086394433536.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2049a4feb80>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This regressor will use default parameters\n",
    "keras_regressor = keras.wrappers.scikit_learn.KerasRegressor(build_model)\n",
    "keras_regressor.fit(x_train, y_train, epochs=4, validation_data=(x_valid, y_valid), callbacks=[keras.callbacks.EarlyStopping(patience=2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 877us/step - loss: 6921085857562624.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-6921085857562624.0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse_test = keras_regressor.score(x_test, y_test)\n",
    "mse_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To hypertune the model\n",
    "from scipy.stats import reciprocal\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 957us/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 799us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 943us/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 778us/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 812us/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 780us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 905us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 974us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 846us/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 715us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 904us/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 939us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 225452395102785186562048.0000 - val_loss: 100215233604248141824.0000\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 54524139506621218816.0000 - val_loss: 25216525126856605696.0000\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 912us/step - loss: 13719596140226150400.0000 - val_loss: 6345084791313924096.0000\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 3452176789902721024.0000 - val_loss: 1596578707022020608.0000\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 864us/step - loss: 868650950863093760.0000 - val_loss: 401736809777725440.0000\n",
      "121/121 [==============================] - 0s 608us/step - loss: 401736809777725440.0000\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: 11709589456158720.0000 - val_loss: 7987096190976.0000\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: 4345545883648.0000 - val_loss: 2009746243584.0000\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 892us/step - loss: 1093444960256.0000 - val_loss: 505701761024.0000\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 611us/step - loss: 275137003520.0000 - val_loss: 127246614528.0000\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 819us/step - loss: 69231067136.0000 - val_loss: 32018307072.0000\n",
      "121/121 [==============================] - 0s 660us/step - loss: 32018292736.0000\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: 14452990547027800092573696.0000 - val_loss: 6175956742849047822336.0000\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: 3360155385229496811520.0000 - val_loss: 1554017716919997562880.0000\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: 845495168362166091776.0000 - val_loss: 391029083849233530880.0000\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 212747126094098857984.0000 - val_loss: 98392384062883889152.0000\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 644us/step - loss: 53532402008599822336.0000 - val_loss: 24757888040585658368.0000\n",
      "121/121 [==============================] - 0s 520us/step - loss: 24757888040585658368.0000\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: 11826264884348798335465816064.0000 - val_loss: 3019063886854333931716608.0000\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 718375157342172964978688.0000 - val_loss: 46377424947914653827072.0000\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 11035341303731766427648.0000 - val_loss: 712428224965924356096.0000\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 169519764875434262528.0000 - val_loss: 10943981290487021568.0000\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 969us/step - loss: 2604082789468864512.0000 - val_loss: 168116307139493888.0000\n",
      "121/121 [==============================] - 0s 780us/step - loss: 168116307139493888.0000\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: 608343883399494192498475008.0000 - val_loss: 1097479959408500994473984.0000\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 261141584681048400199680.0000 - val_loss: 16859023668978126946304.0000\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 4011543931908366794752.0000 - val_loss: 258980129890637971456.0000\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 61623461786798784512.0000 - val_loss: 3978334610357485568.0000\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: 946632026356711424.0000 - val_loss: 61113467611906048.0000\n",
      "121/121 [==============================] - 0s 1ms/step - loss: 61113467611906048.0000\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 784us/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 387us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 755us/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 729us/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 854us/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 745us/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 399us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 969us/step - loss: nan\n",
      "Epoch 1/5\n",
      "242/242 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "242/242 [==============================] - 0s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "242/242 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "242/242 [==============================] - 0s 943us/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "242/242 [==============================] - 0s 718us/step - loss: nan - val_loss: nan\n",
      "121/121 [==============================] - 0s 447us/step - loss: nan\n",
      "Epoch 1/5\n",
      "  1/363 [..............................] - ETA: 52s - loss: 280950.0000"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [            nan             nan -8.38654163e+18             nan\n",
      "             nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 1s 1ms/step - loss: 780.3945 - val_loss: 2.5867\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 537us/step - loss: 1.9242 - val_loss: 1.4510\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 662us/step - loss: 1.4133 - val_loss: 1.3253\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 701us/step - loss: 1.3489 - val_loss: 1.3149\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 723us/step - loss: 1.3407 - val_loss: 1.3155\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3,\n",
       "                   estimator=<keras.wrappers.scikit_learn.KerasRegressor object at 0x00000204993D7880>,\n",
       "                   n_iter=5,\n",
       "                   param_distributions={'learning_rate': <scipy.stats._distn_infrastructure.rv_frozen object at 0x000002049B637C40>,\n",
       "                                        'n_hidden': [1, 2, 3, 4],\n",
       "                                        'n_neurons': [5, 10, 20]})"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {\"n_hidden\":[1,2,3,4], \"n_neurons\":[5,10,20], 'learning_rate':reciprocal(3e-4, 3e-2)}\n",
    "rn_search = RandomizedSearchCV(keras_regressor, param_distributions=params, n_iter=5, cv=3)\n",
    "rn_search.fit(x_train, y_train, epochs=5, validation_data=(x_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'learning_rate': 0.0014234029388933082, 'n_hidden': 1, 'n_neurons': 5}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rn_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.engine.sequential.Sequential at 0x204a1fa0fa0>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model = rn_search.best_estimator_.model\n",
    "best_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 03. Using HyperBand & Bayesian "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(hp):\n",
    "    n_hidden      = hp.Int(\"n_hidden\", min_value=0, max_value=8, default=2)\n",
    "    n_neurons     = hp.Int(\"n_neurons\", min_value=16, max_value=256)\n",
    "    learning_rate = hp.Float(\"learning_rate\", min_value=1e-4, max_value=1e-2,sampling=\"log\")\n",
    "    optimizer     = hp.Choice(\"optimizer\", values=[\"sgd\", \"adam\"])\n",
    "    if optimizer == \"sgd\":\n",
    "        optimizer = keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "    else:\n",
    "        optimizer = keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "    model = keras.Sequential()\n",
    "    model.add(keras.layers.Flatten())\n",
    "    for _ in range(n_hidden):\n",
    "        model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
    "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyClassificationHyperModel(kt.HyperModel):\n",
    "    def build(self, hp):\n",
    "        return build_model(hp)\n",
    "\n",
    "    def fit(self, hp, model, X, y, **kwargs):\n",
    "        if hp.Boolean(\"normalize\"):\n",
    "            norm_layer = keras.layers.Normalization()\n",
    "            X = norm_layer(X)\n",
    "        return model.fit(X, y, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperband_tuner = kt.Hyperband(\n",
    "    MyClassificationHyperModel(), objective=\"val_accuracy\", seed=42,\n",
    "    max_epochs=4, factor=3, hyperband_iterations=2,\n",
    "    overwrite=True, directory=\"california_house\", project_name=\"hyperband\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 20 Complete [00h 00m 03s]\n",
      "val_accuracy: 0.004651162773370743\n",
      "\n",
      "Best val_accuracy So Far: 0.004651162773370743\n",
      "Total elapsed time: 00h 00m 43s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0603 09:26:05.667347 16832 <ipython-input-75-40280a37c112>:4] Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "root_logdir = Path(hyperband_tuner.project_dir) / \"tensorboard\"\n",
    "tensorboard_cb = tf.keras.callbacks.TensorBoard(root_logdir)\n",
    "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=2)\n",
    "hyperband_tuner.search(x_train, y_train, epochs=4, validation_data=(x_valid, y_valid), callbacks=[early_stopping_cb, tensorboard_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 10 Complete [00h 00m 02s]\n",
      "val_accuracy: 0.0036175709683448076\n",
      "\n",
      "Best val_accuracy So Far: 0.004392764996737242\n",
      "Total elapsed time: 00h 00m 31s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0603 09:28:09.447609 16832 <ipython-input-76-ad5ee306d8ba>:6] Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "# Instead of HyperBand you can use BayesianOptimization as well\n",
    "bayesian_opt_tuner = kt.BayesianOptimization(\n",
    "    MyClassificationHyperModel(), objective=\"val_accuracy\", seed=42,\n",
    "    max_trials=10, alpha=1e-4, beta=2.6,\n",
    "    overwrite=True, directory=\"california_house\", project_name=\"bayesian_opt\")\n",
    "bayesian_opt_tuner.search(x_train, y_train, epochs=4,\n",
    "                          validation_data=(x_valid, y_valid),\n",
    "                          callbacks=[early_stopping_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-5fca3220968ba281\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-5fca3220968ba281\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir {root_logdir}"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}