WandB-keras-tuner-BayesianOptimization_LossObjective.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"WandB-keras-tuner-BayesianOptimization_LossMetric.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMfdBE7/JgBvrceaZs1Q69j"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"qLYOKxeTb7CG"},"source":["import tensorflow as tf\r\n","print(tf.__version__)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6fY0J7AwcMh-"},"source":["!pip install keras-tuner\r\n","import kerastuner"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lc4twtsObl7d"},"source":["!pip install wandb -qqq\r\n","import wandb"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VSv8WCHlcFrA"},"source":["!wandb login"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2IhkCR_scIfX","executionInfo":{"status":"ok","timestamp":1615159175889,"user_tz":300,"elapsed":25341,"user":{"displayName":"shakti wadekar","photoUrl":"","userId":"08441724570566508972"}}},"source":["from tensorflow.keras.utils import to_categorical\r\n","from tensorflow.keras.models import Sequential\r\n","from tensorflow.keras.layers import Conv2D\r\n","from tensorflow.keras.layers import MaxPooling2D\r\n","from tensorflow.keras.layers import Dense\r\n","from tensorflow.keras.layers import Flatten\r\n","from tensorflow.keras.optimizers import SGD, Adam\r\n","from tensorflow.keras.layers import BatchNormalization\r\n","from tensorflow.keras.layers import Dropout\r\n","from tensorflow.keras.preprocessing.image import ImageDataGenerator\r\n","import matplotlib.pyplot as plt\r\n","import kerastuner as kt\r\n","from kerastuner.tuners import RandomSearch, Hyperband, BayesianOptimization\r\n","from wandb.keras import WandbCallback"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"5OmBXsVUcVvG"},"source":["(trainX, trainY), (testX, testY) = tf.keras.datasets.cifar10.load_data()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DtwcuB4pcanO","executionInfo":{"status":"ok","timestamp":1615159179630,"user_tz":300,"elapsed":29075,"user":{"displayName":"shakti wadekar","photoUrl":"","userId":"08441724570566508972"}}},"source":["# one hot encode target values\r\n","trainY = to_categorical(trainY)\r\n","testY = to_categorical(testY)"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"8TwL5cl3cci-","executionInfo":{"status":"ok","timestamp":1615159180105,"user_tz":300,"elapsed":29548,"user":{"displayName":"shakti wadekar","photoUrl":"","userId":"08441724570566508972"}}},"source":["# convert from integers to floats\r\n","#train_norm = trainX.astype('float32')\r\n","#test_norm = testX.astype('float32')\r\n","# normalize to range 0-1\r\n","trainX = trainX / 255.0\r\n","testX = testX / 255.0"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"OZii81jucek4","executionInfo":{"status":"ok","timestamp":1615159180107,"user_tz":300,"elapsed":29546,"user":{"displayName":"shakti wadekar","photoUrl":"","userId":"08441724570566508972"}}},"source":["# define cnn model\r\n","def build_model(hp):\r\n","\r\n","  model = Sequential()\r\n","\r\n","  hp_filters = hp.Int('filters', min_value = 32, max_value = 64, step = 32)\r\n","  model.add(Conv2D(filters=hp_filters, kernel_size=(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))\r\n","  \r\n","  model.add(BatchNormalization())\r\n","  model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(MaxPooling2D((2, 2)))\r\n","  model.add(Dropout(0.2))\r\n","  model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(MaxPooling2D((2, 2)))\r\n","  model.add(Dropout(0.3))\r\n","  model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(MaxPooling2D((2, 2)))\r\n","  model.add(Dropout(0.4))\r\n","  model.add(Flatten())\r\n","  model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\r\n","  model.add(BatchNormalization())\r\n","  model.add(Dropout(0.5))\r\n","  model.add(Dense(10, activation='softmax'))\r\n","  \r\n","  hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3])\r\n","  opt = Adam(learning_rate=hp_learning_rate)\r\n","  \r\n","  model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\r\n","  \r\n","  return model"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"id":"2BDIc95XcirV","executionInfo":{"status":"ok","timestamp":1615159180108,"user_tz":300,"elapsed":29544,"user":{"displayName":"shakti wadekar","photoUrl":"","userId":"08441724570566508972"}}},"source":["class MyTuner(kt.Tuner):\r\n","\r\n","    def run_trial(self, trial, trainX, trainY, batch_size, epochs, objective):\r\n","\r\n","        hp = trial.hyperparameters\r\n","        objective_name_str = objective\r\n","\r\n","        ## create the model with the current trial hyperparameters\r\n","        model = self.hypermodel.build(hp)\r\n","        \r\n","        ## Initiates new run for each trial on the dashboard of Weights & Biases\r\n","        run = wandb.init(project=\"WandBAndKerasTuner_BayesianOptimization\", config=hp.values)\r\n","\r\n","        ## WandbCallback() logs all the metric data such as\r\n","        ## loss, accuracy and etc on dashboard for visualization\r\n","        history = model.fit(trainX,\r\n","                  trainY,\r\n","                  batch_size=batch_size,\r\n","                  epochs=epochs,\r\n","                  validation_split=0.1,\r\n","                  callbacks=[WandbCallback()])  \r\n","\r\n","        ## if loss used, use the loss of last epoch model which is fully trained\r\n","        training_loss = history.history['loss'][-1]  ## [-1] will give the last value in the list\r\n","\r\n","        ## Send the objective data to the oracle for comparison of hyperparameters\r\n","        self.oracle.update_trial(trial.trial_id, {objective_name_str:training_loss})\r\n","\r\n","        ## save the trial model\r\n","        self.save_model(trial.trial_id, model)\r\n","        \r\n","        ## ends the run on the Weights & Biases dashboard\r\n","        run.finish()\r\n","\r\n","        "],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"T1n29SM6ee9n"},"source":["## set the objective of tuning algorithm\r\n","objective = 'loss'  ## training_loss\r\n","  \r\n","## instantiate the new Tuner with tuning algorithm and required parameters\r\n","tuner = MyTuner(\r\n","      oracle=kt.oracles.BayesianOptimization(\r\n","          objective=objective,\r\n","          max_trials=4),\r\n","      hypermodel=build_model,\r\n","      directory='./results_of_WandBwithKerasTuner9')\r\n","\r\n","tuner.search_space_summary()\r\n","\r\n","## initiates the hyperparameter tuning process\r\n","tuner.search(trainX, trainY, batch_size=32, epochs=5, objective=objective)\r\n","\r\n","## get the best hyperparameters\r\n","best_hps = tuner.get_best_hyperparameters()[0]\r\n","print(best_hps.values)\r\n","\r\n","## get the best\r\n","best_model = tuner.get_best_models()[0]\r\n"],"execution_count":null,"outputs":[]}]}