Deep Learning/04. Complex model using Keras Functional API.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functional API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building a complex model using the Functional API\n",
    "\n",
    "##### Not all Neural network are simply sequential. Some have complex topologies, some have multiple input/output. For ex deep & wide NN connect its some input direct to the output."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lets load the california dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (3870, 8) (5160, 8)\n"
     ]
    }
   ],
   "source": [
    "housing = datasets.fetch_california_housing()\n",
    "x_train, x_test, y_train, y_test   = train_test_split(housing.data, housing.target, random_state=42)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, random_state=42)\n",
    "print(x_train.shape, x_valid.shape, x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset the name counter\n",
    "def cls():\n",
    "    tf.keras.backend.clear_session()\n",
    "    tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 01. Network with one input and one output. Input alo connect with output directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 8)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "normalization (Normalization)   (None, 8)            17          input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 30)           270         normalization[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 38)           0           input_1[0][0]                    \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            39          concatenate[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 1,256\n",
      "Trainable params: 1,239\n",
      "Non-trainable params: 17\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cls()\n",
    "normalization_layer = keras.layers.Normalization()\n",
    "input_  = keras.layers.Input(shape=x_train.shape[1:])\n",
    "normal  = normalization_layer(input_)\n",
    "hidden1 = keras.layers.Dense(30, activation='relu')(normal)\n",
    "hidden2 = keras.layers.Dense(30, activation='relu')(hidden1)\n",
    "concat  = keras.layers.Concatenate()([input_, hidden2])\n",
    "output_ = keras.layers.Dense(1)(concat)\n",
    "model   = keras.Model(inputs=[input_], outputs=[output_])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 122.3226 - root_mean_squared_error: 11.0600 - val_loss: 305.9134 - val_root_mean_squared_error: 17.4904\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 5.5425 - root_mean_squared_error: 2.3543 - val_loss: 183.4623 - val_root_mean_squared_error: 13.5448\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 3.0631 - root_mean_squared_error: 1.7502 - val_loss: 87.2228 - val_root_mean_squared_error: 9.3393\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 1.5796 - root_mean_squared_error: 1.2568 - val_loss: 35.3699 - val_root_mean_squared_error: 5.9473\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.9536 - root_mean_squared_error: 0.9765 - val_loss: 12.3882 - val_root_mean_squared_error: 3.5197\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=keras.optimizers.Adam(learning_rate=1e-3), metrics=[keras.metrics.RootMeanSquaredError()])\n",
    "normalization_layer.adapt(x_train)\n",
    "history = model.fit(x_train, y_train, epochs=5, validation_data=(x_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 1ms/step - loss: 0.5481 - root_mean_squared_error: 0.7404\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.548128604888916, 0.7403571009635925]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 02. Network with multiple input and single output\n",
    "Support I want to pass 4 starting features directly to the last layer and rest of the features or overlap through the hidden layers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "deep_input (InputLayer)         [(None, 6)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "normalization_1 (Normalization) (None, 6)            13          deep_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "wide_input (InputLayer)         [(None, 5)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 30)           210         normalization_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "normalization (Normalization)   (None, 5)            11          wide_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 35)           0           normalization[0][0]              \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 1)            36          concatenate[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 1,200\n",
      "Trainable params: 1,176\n",
      "Non-trainable params: 24\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cls()\n",
    "input_wide  = keras.layers.Input(shape=[5], name='wide_input') # 5 features [0 to 4] directly to the output layer\n",
    "input_deep  = keras.layers.Input(shape=[6], name='deep_input') # 6 features [2 to 7] through hidden layers.\n",
    "normal_layer_wide = keras.layers.Normalization()\n",
    "normal_layer_deep = keras.layers.Normalization()\n",
    "normal_wide = normal_layer_wide(input_wide) \n",
    "normal_deep = normal_layer_deep(input_deep)\n",
    "hidden1 = keras.layers.Dense(30, activation='relu')(normal_deep)\n",
    "hidden2 = keras.layers.Dense(30, activation='relu')(hidden1)\n",
    "concat = keras.layers.Concatenate()([normal_wide, hidden2])\n",
    "output = keras.layers.Dense(1, name='output')(concat)\n",
    "model = keras.Model(inputs=[input_wide, input_deep], outputs=[output])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.2768 - root_mean_squared_error: 1.1300 - val_loss: 0.9497 - val_root_mean_squared_error: 0.9745\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4767 - root_mean_squared_error: 0.6904 - val_loss: 1.4311 - val_root_mean_squared_error: 1.1963\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4433 - root_mean_squared_error: 0.6658 - val_loss: 0.4258 - val_root_mean_squared_error: 0.6525\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4057 - root_mean_squared_error: 0.6370 - val_loss: 0.4016 - val_root_mean_squared_error: 0.6338\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3940 - root_mean_squared_error: 0.6277 - val_loss: 1.4914 - val_root_mean_squared_error: 1.2212\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=keras.losses.MeanSquaredError(), optimizer=keras.optimizers.Adam(learning_rate=1e-3), metrics=keras.metrics.RootMeanSquaredError())\n",
    "x_train_wide, x_train_deep = x_train[:,:5], x_train[:,2:]\n",
    "x_valid_wide, x_valid_deep = x_valid[:,:5], x_valid[:,2:]\n",
    "x_test_wide , x_test_deep  = x_test[:,:5] , x_test[:,2:]\n",
    "normal_layer_wide.adapt(x_train_wide)\n",
    "normal_layer_deep.adapt(x_train_deep)\n",
    "history = model.fit({'wide_input':x_train_wide, \"deep_input\":x_train_deep}, y_train, epochs=5, validation_data=((x_valid_wide, x_valid_deep), y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 1ms/step - loss: 0.3865 - root_mean_squared_error: 0.6217\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.38654041290283203, 0.6217237710952759]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate((x_test_wide,x_test_deep), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.35351694]], dtype=float32), array([0.477]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict((x_test_wide[:1], x_test_deep[:1])), y_test[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 03. Handling multiple output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "deep_input (InputLayer)         [(None, 6)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "normalization_1 (Normalization) (None, 6)            13          deep_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "wide_input (InputLayer)         [(None, 5)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 30)           210         normalization_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "normalization (Normalization)   (None, 5)            11          wide_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 35)           0           normalization[0][0]              \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "main_output (Dense)             (None, 1)            36          concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "aux_output (Dense)              (None, 1)            31          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,207\n",
      "Non-trainable params: 24\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cls()\n",
    "input_wide  = keras.layers.Input(shape=[5], name='wide_input') # 5 features [0 to 4] directly to the output layer\n",
    "input_deep  = keras.layers.Input(shape=[6], name='deep_input') # 6 features [2 to 7] through hidden layers.\n",
    "normal_layer_wide = keras.layers.Normalization()\n",
    "normal_layer_deep = keras.layers.Normalization()\n",
    "normal_wide = normal_layer_wide(input_wide) \n",
    "normal_deep = normal_layer_deep(input_deep)\n",
    "hidden1 = keras.layers.Dense(30, activation='relu')(normal_deep)\n",
    "hidden2 = keras.layers.Dense(30, activation='relu')(hidden1)\n",
    "concat = keras.layers.Concatenate()([normal_wide, hidden2])\n",
    "output = keras.layers.Dense(1, name='main_output')(concat)\n",
    "aux_output = keras.layers.Dense(1, name='aux_output')(hidden2) \n",
    "model = keras.Model(inputs=[input_wide, input_deep], outputs=[output, aux_output])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.3490 - main_output_loss: 1.2742 - aux_output_loss: 2.0215 - main_output_root_mean_squared_error: 1.1288 - aux_output_root_mean_squared_error: 1.4218 - val_loss: 1.5415 - val_main_output_loss: 0.9593 - val_aux_output_loss: 6.7806 - val_main_output_root_mean_squared_error: 0.9795 - val_aux_output_root_mean_squared_error: 2.6040\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.5101 - main_output_loss: 0.4785 - aux_output_loss: 0.7952 - main_output_root_mean_squared_error: 0.6917 - aux_output_root_mean_squared_error: 0.8917 - val_loss: 1.3624 - val_main_output_loss: 1.0094 - val_aux_output_loss: 4.5401 - val_main_output_root_mean_squared_error: 1.0047 - val_aux_output_root_mean_squared_error: 2.1307\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4618 - main_output_loss: 0.4404 - aux_output_loss: 0.6546 - main_output_root_mean_squared_error: 0.6636 - aux_output_root_mean_squared_error: 0.8091 - val_loss: 0.5361 - val_main_output_loss: 0.3975 - val_aux_output_loss: 1.7837 - val_main_output_root_mean_squared_error: 0.6305 - val_aux_output_root_mean_squared_error: 1.3355\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4252 - main_output_loss: 0.4059 - aux_output_loss: 0.5985 - main_output_root_mean_squared_error: 0.6371 - aux_output_root_mean_squared_error: 0.7736 - val_loss: 0.5182 - val_main_output_loss: 0.4590 - val_aux_output_loss: 1.0517 - val_main_output_root_mean_squared_error: 0.6775 - val_aux_output_root_mean_squared_error: 1.0255\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4106 - main_output_loss: 0.3931 - aux_output_loss: 0.5690 - main_output_root_mean_squared_error: 0.6269 - aux_output_root_mean_squared_error: 0.7543 - val_loss: 0.4049 - val_main_output_loss: 0.3588 - val_aux_output_loss: 0.8196 - val_main_output_root_mean_squared_error: 0.5990 - val_aux_output_root_mean_squared_error: 0.9053\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss={\"main_output\":\"mse\", \"aux_output\":\"mse\"}, loss_weights=(0.9, 0.1), optimizer=keras.optimizers.Adam(learning_rate=1e-3), metrics=keras.metrics.RootMeanSquaredError())\n",
    "normal_layer_wide.adapt(x_train_wide)\n",
    "normal_layer_deep.adapt(x_train_deep)\n",
    "history = model.fit((x_train_wide, x_train_deep), (y_train, y_train), epochs=5, validation_data=((x_valid_wide, x_valid_deep), (y_valid,y_valid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 1ms/step - loss: 0.3963 - main_output_loss: 0.3808 - aux_output_loss: 0.5356 - main_output_root_mean_squared_error: 0.6171 - aux_output_root_mean_squared_error: 0.7318\n",
      "0.396294504404068 0.380817174911499 0.535590648651123 0.6171038746833801 0.7318406105041504\n"
     ]
    }
   ],
   "source": [
    "weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse = model.evaluate((x_test_wide,x_test_deep), (y_test,y_test))\n",
    "print(weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([array([[0.5382582]], dtype=float32), array([[0.67674875]], dtype=float32)],\n",
       " array([0.477]))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict((x_test_wide[:1], x_test_deep[:1])), y_test[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "deep_input (InputLayer)         [(None, 6)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "normalization_1 (Normalization) (None, 6)            13          deep_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "wide_input (InputLayer)         [(None, 5)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 30)           210         normalization_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "normalization (Normalization)   (None, 5)            11          wide_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 35)           0           normalization[0][0]              \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "main_output (Dense)             (None, 1)            36          concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "aux_output (Dense)              (None, 1)            31          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,207\n",
      "Non-trainable params: 24\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model successfully saved and you can read as well\n",
    "model.save(\"multi_input_multi_output.h5\")\n",
    "keras.models.load_model('multi_input_multi_output.h5').summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building Dynamic Models using SubClassing API\n",
    "Both the sequential API and the Functional API are declarative: you start by declaring which layers you want to use and how they should be connected, and only then can you start feeding the data to the model for training or inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WideAndDeepModel(keras.Model):\n",
    "    def __init__(self, units=30, activation=\"relu\", **kwargs):\n",
    "        super().__init__(**kwargs)  # needed to support naming the model\n",
    "        self.norm_layer_wide = keras.layers.Normalization()\n",
    "        self.norm_layer_deep = keras.layers.Normalization()\n",
    "        self.hidden1 = keras.layers.Dense(units, activation=activation)\n",
    "        self.hidden2 = keras.layers.Dense(units, activation=activation)\n",
    "        self.main_output = keras.layers.Dense(1)\n",
    "        self.aux_output  = keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        input_wide, input_deep = inputs\n",
    "        norm_wide = self.norm_layer_wide(input_wide)\n",
    "        norm_deep = self.norm_layer_deep(input_deep)\n",
    "        hidden1 = self.hidden1(norm_deep)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([norm_wide, hidden2])\n",
    "        output = self.main_output(concat)\n",
    "        aux_output = self.aux_output(hidden2)\n",
    "        return output, aux_output\n",
    "\n",
    "cls()\n",
    "model = WideAndDeepModel(30, activation=\"relu\", name=\"my_cool_model\")\n",
    "\n",
    "# You can do anything in call() for loops, if statements, low-level tensorflow operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.3490 - output_1_loss: 1.2742 - output_2_loss: 2.0215 - output_1_root_mean_squared_error: 1.1288 - output_2_root_mean_squared_error: 1.4218 - val_loss: 1.5415 - val_output_1_loss: 0.9593 - val_output_2_loss: 6.7806 - val_output_1_root_mean_squared_error: 0.9795 - val_output_2_root_mean_squared_error: 2.6040\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5101 - output_1_loss: 0.4785 - output_2_loss: 0.7952 - output_1_root_mean_squared_error: 0.6917 - output_2_root_mean_squared_error: 0.8917 - val_loss: 1.3624 - val_output_1_loss: 1.0094 - val_output_2_loss: 4.5401 - val_output_1_root_mean_squared_error: 1.0047 - val_output_2_root_mean_squared_error: 2.1307\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4618 - output_1_loss: 0.4404 - output_2_loss: 0.6546 - output_1_root_mean_squared_error: 0.6636 - output_2_root_mean_squared_error: 0.8091 - val_loss: 0.5361 - val_output_1_loss: 0.3975 - val_output_2_loss: 1.7837 - val_output_1_root_mean_squared_error: 0.6305 - val_output_2_root_mean_squared_error: 1.3355\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.4252 - output_1_loss: 0.4059 - output_2_loss: 0.5985 - output_1_root_mean_squared_error: 0.6371 - output_2_root_mean_squared_error: 0.7736 - val_loss: 0.5182 - val_output_1_loss: 0.4590 - val_output_2_loss: 1.0517 - val_output_1_root_mean_squared_error: 0.6775 - val_output_2_root_mean_squared_error: 1.0255\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.4106 - output_1_loss: 0.3931 - output_2_loss: 0.5690 - output_1_root_mean_squared_error: 0.6269 - output_2_root_mean_squared_error: 0.7543 - val_loss: 0.4049 - val_output_1_loss: 0.3588 - val_output_2_loss: 0.8196 - val_output_1_root_mean_squared_error: 0.5990 - val_output_2_root_mean_squared_error: 0.9053\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=(\"mse\",\"mse\"), loss_weights=(0.9, 0.1), optimizer=keras.optimizers.Adam(learning_rate=1e-3), metrics=keras.metrics.RootMeanSquaredError())\n",
    "model.norm_layer_wide.adapt(x_train_wide)\n",
    "model.norm_layer_deep.adapt(x_train_deep)\n",
    "history = model.fit((x_train_wide, x_train_deep), (y_train, y_train), epochs=5, validation_data=((x_valid_wide, x_valid_deep), (y_valid,y_valid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"my_cool_model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "normalization (Normalization multiple                  11        \n",
      "_________________________________________________________________\n",
      "normalization_1 (Normalizati multiple                  13        \n",
      "_________________________________________________________________\n",
      "dense (Dense)                multiple                  210       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              multiple                  930       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              multiple                  36        \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              multiple                  31        \n",
      "=================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,207\n",
      "Non-trainable params: 24\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Keras will not know about what is placed inside call() so it would let us know the connections of layers, shapes, dtypes, etc\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 920us/step - loss: 0.3963 - output_1_loss: 0.3808 - output_2_loss: 0.5356 - output_1_root_mean_squared_error: 0.6171 - output_2_root_mean_squared_error: 0.7318\n",
      "0.396294504404068 0.380817174911499 0.535590648651123 0.6171038746833801 0.7318406105041504\n"
     ]
    }
   ],
   "source": [
    "weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse = model.evaluate((x_test_wide,x_test_deep), (y_test,y_test))\n",
    "print(weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 10 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C8EE59E550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((array([[0.5382582]], dtype=float32), array([[0.67674875]], dtype=float32)),\n",
       " array([0.477]))"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict((x_test_wide[:1], x_test_deep[:1])), y_test[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "ename": "NotImplementedError",
     "evalue": "Saving the model to HDF5 format requires the model to be a Functional model or a Sequential model. It does not work for subclassed models, because such models are defined via the body of a Python method, which isn't safely serializable. Consider saving to the Tensorflow SavedModel format (by setting save_format=\"tf\") or using `save_weights`.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNotImplementedError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-76-e8dd42fc3349>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Now it will not save to .h5 format directly because it's not functional or sequential model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"multi_input_multi_output.h5\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, filepath, overwrite, include_optimizer, save_format, signatures, options, save_traces)\u001b[0m\n\u001b[0;32m   2143\u001b[0m     \"\"\"\n\u001b[0;32m   2144\u001b[0m     \u001b[1;31m# pylint: enable=line-too-long\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2145\u001b[1;33m     save.save_model(self, filepath, overwrite, include_optimizer, save_format,\n\u001b[0m\u001b[0;32m   2146\u001b[0m                     signatures, options, save_traces)\n\u001b[0;32m   2147\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\saving\\save.py\u001b[0m in \u001b[0;36msave_model\u001b[1;34m(model, filepath, overwrite, include_optimizer, save_format, signatures, options, save_traces)\u001b[0m\n\u001b[0;32m    136\u001b[0m     if (not model._is_graph_network and  # pylint:disable=protected-access\n\u001b[0;32m    137\u001b[0m         not isinstance(model, sequential.Sequential)):\n\u001b[1;32m--> 138\u001b[1;33m       raise NotImplementedError(\n\u001b[0m\u001b[0;32m    139\u001b[0m           \u001b[1;34m'Saving the model to HDF5 format requires the model to be a '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m           \u001b[1;34m'Functional model or a Sequential model. It does not work for '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNotImplementedError\u001b[0m: Saving the model to HDF5 format requires the model to be a Functional model or a Sequential model. It does not work for subclassed models, because such models are defined via the body of a Python method, which isn't safely serializable. Consider saving to the Tensorflow SavedModel format (by setting save_format=\"tf\") or using `save_weights`."
     ]
    }
   ],
   "source": [
    "# Now it will not save to .h5 format directly because it's not functional or sequential model\n",
    "model.save(\"multi_input_multi_output.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: multi_input_multi_output\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"multi_input_multi_output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_input_multi_output\\assets\n",
      "multi_input_multi_output\\keras_metadata.pb\n",
      "multi_input_multi_output\\saved_model.pb\n",
      "multi_input_multi_output\\variables\n",
      "multi_input_multi_output\\variables\\variables.data-00000-of-00001\n",
      "multi_input_multi_output\\variables\\variables.index\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None, None]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To check how many directories created\n",
    "[print(i) for i in sorted(Path(\"multi_input_multi_output\").glob(\"**/*\"))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((array([[0.5382582]], dtype=float32), array([[0.67674875]], dtype=float32)),\n",
       " array([0.477]))"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.load_model(\"multi_input_multi_output\")\n",
    "\n",
    "model.predict((x_test_wide[:1], x_test_deep[:1])), y_test[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(\"weights/multi_input_multi_output_weights\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x1c8fe2e3e50>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_weights('weights/multi_input_multi_output_weights') # First build the model the load the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the directory recursively\n",
    "import shutil\n",
    "shutil.rmtree(\"weights\", ignore_errors=True)\n",
    "shutil.rmtree(\"multi_input_multi_output\", ignore_errors=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Miscellaneous\n",
    "To improve the performance of the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 01. Callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.3865 - output_1_loss: 0.3724 - output_2_loss: 0.5140 - output_1_root_mean_squared_error: 0.6102 - output_2_root_mean_squared_error: 0.7169 - val_loss: 1.3805 - val_output_1_loss: 1.3943 - val_output_2_loss: 1.2569 - val_output_1_root_mean_squared_error: 1.1808 - val_output_2_root_mean_squared_error: 1.1211\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3866 - output_1_loss: 0.3747 - output_2_loss: 0.4940 - output_1_root_mean_squared_error: 0.6121 - output_2_root_mean_squared_error: 0.7028 - val_loss: 1.5084 - val_output_1_loss: 1.6232 - val_output_2_loss: 0.4755 - val_output_1_root_mean_squared_error: 1.2741 - val_output_2_root_mean_squared_error: 0.6896\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3915 - output_1_loss: 0.3835 - output_2_loss: 0.4631 - output_1_root_mean_squared_error: 0.6193 - output_2_root_mean_squared_error: 0.6805 - val_loss: 2.2948 - val_output_1_loss: 2.3573 - val_output_2_loss: 1.7322 - val_output_1_root_mean_squared_error: 1.5354 - val_output_2_root_mean_squared_error: 1.3161\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3757 - output_1_loss: 0.3668 - output_2_loss: 0.4558 - output_1_root_mean_squared_error: 0.6056 - output_2_root_mean_squared_error: 0.6751 - val_loss: 1.9265 - val_output_1_loss: 2.0827 - val_output_2_loss: 0.5211 - val_output_1_root_mean_squared_error: 1.4431 - val_output_2_root_mean_squared_error: 0.7219\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3715 - output_1_loss: 0.3643 - output_2_loss: 0.4367 - output_1_root_mean_squared_error: 0.6035 - output_2_root_mean_squared_error: 0.6608 - val_loss: 2.3224 - val_output_1_loss: 2.3934 - val_output_2_loss: 1.6838 - val_output_1_root_mean_squared_error: 1.5471 - val_output_2_root_mean_squared_error: 1.2976\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n"
     ]
    }
   ],
   "source": [
    "# We are using the same model object()\n",
    "\n",
    "# Save the last trained epoch \n",
    "cb_checkpoint = keras.callbacks.ModelCheckpoint(\"model/checkpoint\") # save_best_only=True will save the best model where validation score low\n",
    "\n",
    "history = model.fit((x_train_wide, x_train_deep), (y_train, y_train), epochs=5, \n",
    "                    validation_data=((x_valid_wide, x_valid_deep), (y_valid,y_valid)),\n",
    "                    callbacks=[cb_checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.3431 - output_1_loss: 0.3368 - output_2_loss: 0.4006 - output_1_root_mean_squared_error: 0.5803 - output_2_root_mean_squared_error: 0.6329 - val_loss: 0.9825 - val_output_1_loss: 1.0124 - val_output_2_loss: 0.7126 - val_output_1_root_mean_squared_error: 1.0062 - val_output_2_root_mean_squared_error: 0.8442\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3463 - output_1_loss: 0.3402 - output_2_loss: 0.4006 - output_1_root_mean_squared_error: 0.5833 - output_2_root_mean_squared_error: 0.6329 - val_loss: 0.8435 - val_output_1_loss: 0.8879 - val_output_2_loss: 0.4438 - val_output_1_root_mean_squared_error: 0.9423 - val_output_2_root_mean_squared_error: 0.6662\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3487 - output_1_loss: 0.3435 - output_2_loss: 0.3962 - output_1_root_mean_squared_error: 0.5861 - output_2_root_mean_squared_error: 0.6294 - val_loss: 1.3399 - val_output_1_loss: 1.3241 - val_output_2_loss: 1.4821 - val_output_1_root_mean_squared_error: 1.1507 - val_output_2_root_mean_squared_error: 1.2174\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3442 - output_1_loss: 0.3380 - output_2_loss: 0.4000 - output_1_root_mean_squared_error: 0.5814 - output_2_root_mean_squared_error: 0.6325 - val_loss: 1.8434 - val_output_1_loss: 1.9630 - val_output_2_loss: 0.7671 - val_output_1_root_mean_squared_error: 1.4011 - val_output_2_root_mean_squared_error: 0.8758\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.3476 - output_1_loss: 0.3424 - output_2_loss: 0.3946 - output_1_root_mean_squared_error: 0.5851 - output_2_root_mean_squared_error: 0.6282 - val_loss: 2.5864 - val_output_1_loss: 2.5739 - val_output_2_loss: 2.6987 - val_output_1_root_mean_squared_error: 1.6043 - val_output_2_root_mean_squared_error: 1.6428\n",
      "INFO:tensorflow:Assets written to: model\\checkpoint\\assets\n"
     ]
    }
   ],
   "source": [
    "cb_earlystopping = keras.callbacks.EarlyStopping(patience=3)\n",
    "history = model.fit((x_train_wide, x_train_deep), (y_train, y_train), epochs=5, \n",
    "                    validation_data=((x_valid_wide, x_valid_deep), (y_valid,y_valid)),\n",
    "                    callbacks=[cb_checkpoint, cb_earlystopping])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 02. Custom callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrintValTrainRatioCallback(keras.callbacks.Callback):\n",
    "    # You can also implement\n",
    "    # on_train_begin(), on_train_end(), on_epoch_begin(), on_epoch_end(), on_batch_begin(), on_batch_end(), on_test_begin(), on_test_end()\n",
    "    # on_test_batch_begin(), on_predict_begin(), on_predict_batch_begin()\n",
    "    def on_epoch_end(self, epoch, logs):\n",
    "        print(\"\\nval/train: {:.2f}\".format(logs['val_loss'] / logs['loss']))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val/train: 0.99\n",
      "\n",
      "val/train: 1.07\n",
      "\n",
      "val/train: 0.95\n",
      "\n",
      "val/train: 0.99\n",
      "\n",
      "val/train: 1.06\n"
     ]
    }
   ],
   "source": [
    "cb_val_train_ratio = PrintValTrainRatioCallback()\n",
    "history = model.fit((x_train_wide, x_train_deep), (y_train, y_train), epochs=5, \n",
    "                    validation_data=((x_valid_wide, x_valid_deep), (y_valid,y_valid)),\n",
    "                    callbacks=[cb_val_train_ratio], verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}