Homework5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the iris dataset\n",
    "\n",
    "iris = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store predictive features and the X and y variable\n",
    "\n",
    "X = iris.data\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the dataset into the training set and test set and take necessary transformations \n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, stratify=y)\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "ss = StandardScaler()\n",
    "X_test_ss = ss.fit_transform(X_test)\n",
    "X_train_ss = ss.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function called make_model for GridSearch\n",
    "\n",
    "def make_model(optimizer=\"adam\", hidden_size=5,drop_out=0.0):\n",
    "    model = Sequential([\n",
    "      Dense(hidden_size, input_shape=(4,)),\n",
    "      Activation('relu'),\n",
    "      Dense(8),\n",
    "      Dropout(drop_out),\n",
    "      Activation('relu'),\n",
    "      Dense(3),\n",
    "      Dropout(drop_out),\n",
    "      Activation(\"softmax\")\n",
    "    ])\n",
    "    model.compile(optimizer=optimizer,loss=\"categorical_crossentropy\",\n",
    "                  metrics=['accuracy'])\n",
    "    return model\n",
    "clf = KerasClassifier(make_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the classifier to the data\n",
    "\n",
    "clf.fit(X_train_ss, y_train, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GridSearch for the best tuning parameters\n",
    "\n",
    "param_grid = {'epochs': np.arange(50,80,10),  \n",
    "              'hidden_size': [64,128,256],\n",
    "              'drop_out' : [0.1, 0.25, 0.5]\n",
    "             }\n",
    "\n",
    "\n",
    "cv = StratifiedShuffleSplit(n_splits=2, test_size=0.5, random_state=42)\n",
    "grid = GridSearchCV(clf, param_grid=param_grid, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fit the model to the data\n",
    "\n",
    "grid.fit(X_train_ss, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe to show model performances depended on tuning parameters\n",
    "\n",
    "res = pd.DataFrame(grid.cv_results_)\n",
    "res.pivot_table(index=[\"param_epochs\", \"param_hidden_size\",\"param_drop_out\" ],\n",
    "                values=['mean_train_score', \"mean_test_score\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract the best tuning parameter combination \n",
    "\n",
    "grid.best_params_\n",
    "\n",
    "# Store the best test scoret\n",
    "\n",
    "score = grid.score(X_test_ss, y_test)\n",
    "\n",
    "# Print the best test score \n",
    "\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "\n",
    "from sklearn import datasets\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.datasets import fashion_mnist\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "device_name = tf.test.gpu_device_name()\n",
    "if device_name != '/device:GPU:0':\n",
    "  raise SystemError('GPU device not found')\n",
    "print('Found GPU at: {}'.format(device_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "# 60000 samples, 28 x 28 pixel\n",
    "\n",
    "((X_train, y_train), (X_test, y_test)) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a random seed for subsampling\n",
    "\n",
    "idx = np.random.randint(1,60000,size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Subsample 10,000 samples out of 60,000 samples\n",
    "\n",
    "X_sub_train = X_train[idx]\n",
    "y_sub_train = y_train[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if a corresponding matrix is aligned correctly by visualization \n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "\n",
    "plt.imshow(X_sub_train[0], cmap=plt.get_cmap('gray'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape matrices from 3D into 2D\n",
    "\n",
    "X_sub_train = X_sub_train.reshape(X_sub_train.shape[0], (X_sub_train.shape[1]*X_sub_train.shape[2]))\n",
    "X_sub_test = X_test.reshape(X_test.shape[0],(X_test.shape[1]*X_test.shape[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change numerics into a matrix form for the target feature\n",
    "\n",
    "n_classes = len(set(y_sub_train))\n",
    "y_sub_train = to_categorical(y_sub_train, num_classes = n_classes)\n",
    "y_sub_test = to_categorical(y_test, num_classes = n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the dataset into the training set and test set\n",
    "\n",
    "X_sub_train, X_sub_val, y_sub_train, y_sub_val = train_test_split(X_sub_train, y_sub_train, stratify = y_sub_train, test_size = 0.3, random_state=42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vanila Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a 2-layer dense neural network\n",
    "\n",
    "model = Sequential([\n",
    "    Dense(32, input_shape = (784,)),\n",
    "    Activation(\"relu\"),\n",
    "    Dense(16),\n",
    "    Activation(\"relu\"),\n",
    "    Dense(12),\n",
    "    Activation(\"relu\"),\n",
    "    Dense(10),\n",
    "    Activation(\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configure the model for training\n",
    "\n",
    "model.compile(optimizer = \"adam\", loss=\"categorical_crossentropy\", metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the model to the training data\n",
    "\n",
    "model.fit(X_sub_train, y_sub_train, validation_split=0.3, batch_size = 32, epochs = 75, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "\n",
    "score = model.evaluate(X_sub_test, y_sub_test, verbose=0)\n",
    "model1_loss = score[0]\n",
    "model1_acc = score[1]\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a learning curve\n",
    "\n",
    "\n",
    "epochs = 75\n",
    "\n",
    "def learning_curve(model, X_train, y_train, epochs):\n",
    "  \n",
    "  \n",
    "  model.compile(optimizer = \"adam\", loss=\"categorical_crossentropy\", metrics=['accuracy'])\n",
    "  model_hist = model.fit(X_train, y_train, validation_split=0.3, batch_size = 32, epochs = epochs, verbose = 0)\n",
    "  \n",
    "  plt.plot(model_hist.history['acc'])\n",
    "  plt.plot(model_hist.history['val_acc'])\n",
    "  plt.title(\"Accuracy vs Epoch\")\n",
    "  plt.ylabel(\"Accuracy\")\n",
    "  plt.xlabel(\"Number of Epoch\")\n",
    "  plt.xticks(np.arange(0, epochs+1, 10))\n",
    "  plt.legend(['train', 'test'], loc='upper right')\n",
    "  plt.show()\n",
    "  \n",
    "  return plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_curve(model,X_sub_train, y_sub_train, epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vanilla model using drop-out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the model using drop-out\n",
    "\n",
    "from keras.layers import Dropout\n",
    "\n",
    "dropout_model = Sequential ([\n",
    "    Dense(100, input_shape = (784,), activation='relu'),\n",
    "    Dropout(.25),\n",
    "    Dense(50, activation='relu'),\n",
    "    Dropout(.25),\n",
    "    Dense(25, activation='relu'),\n",
    "    Dropout(.25),\n",
    "    Dense(10, activation='softmax'),\n",
    "    \n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile and fit the model to the data\n",
    "\n",
    "dropout_model.compile(optimizer = \"adam\", loss=\"categorical_crossentropy\", \n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "dropout_model.fit(X_sub_train, y_sub_train, validation_split=0.3, batch_size = 32, epochs = 75, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and print the loss and accuracy\n",
    "\n",
    "score = dropout_model.evaluate(X_sub_val, y_sub_val, verbose=0)\n",
    "model2_loss = score[0]\n",
    "model2_acc = score[1]\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Draw a learning curve\n",
    "\n",
    "epochs = 75\n",
    "learning_curve(dropout_model, X_sub_train, y_sub_train, epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model using batch normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data preprocessing\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "X_train_img = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2],1)\n",
    "X_test_img = X_test.reshape(X_test.shape[0], X_test.shape[1], X_train.shape[2],1)\n",
    "\n",
    "\n",
    "n_classes = len(set(y_train))\n",
    "y_train_img = to_categorical(y_train, n_classes)\n",
    "y_test_img = to_categorical(y_test, n_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into the training set and test set\n",
    "\n",
    "X_train_bn, X_test_bn, y_train_bn, y_test_bn = train_test_split(X_train_img, y_train_img, stratify = y_train_img,\n",
    "                                                               test_size = 0.3, random_state=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building the model with batch normaliation \n",
    "\n",
    "from keras.layers import BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten\n",
    "\n",
    "input_shape = (28,28,1)\n",
    "\n",
    "cnv_bn = Sequential([\n",
    "    Conv2D(8, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu'),\n",
    "    BatchNormalization(),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Conv2D(8, kernel_size = (3,3),\n",
    "          input_shape = (784,), activation = 'relu'),\n",
    "    BatchNormalization(),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Flatten(),\n",
    "    Dense(32, activation='relu'),\n",
    "    Dense(10, activation = 'softmax')\n",
    "    \n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complile and fit the model to the data\n",
    "\n",
    "cnv_bn.compile(optimizer = \"adam\", loss=\"categorical_crossentropy\", \n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "cnv_bn.fit(X_train_bn, y_train_bn, validation_split=0.3, batch_size = 32, epochs = 75, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and print loss and accuracy \n",
    "\n",
    "score = cnv_bn.evaluate(X_test_bn, y_test_bn, verbose=0)\n",
    "model3_loss = score[0]\n",
    "model3_acc = score[1]\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Draw a learning curve\n",
    "\n",
    "epochs = 75\n",
    "learning_curve(cnv_bn, X_train_bn, y_train_bn, epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model using residual connections without drop-out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Preprocessing\n",
    "\n",
    "X_train_rcnn = X_train.reshape(X_train.shape[0], (X_train.shape[1]* X_train.shape[2]))\n",
    "X_test_rcnn = X_test.reshape(X_test.shape[0], (X_test.shape[1]* X_test.shape[2]))\n",
    "\n",
    "\n",
    "n_classes = len(set(y_train))\n",
    "y_train_rcnn = to_categorical(y_train, n_classes)\n",
    "y_test_rcnn = to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into the training and test set\n",
    "\n",
    "X_train_rcnn_fit, X_test_rcnn_fit, y_train_rcnn_fit, y_test_rcnn_fit = train_test_split(X_train_rcnn, y_train_rcnn, stratify = y_train_rcnn,\n",
    "                                                                                       test_size = 0.3, random_state=17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the resnet model\n",
    "\n",
    "from keras.layers import Input, Dense\n",
    "from keras.models import Model\n",
    "\n",
    "inputs = Input(shape=(784,))\n",
    "\n",
    "x = Dense(64, activation='relu')(inputs)\n",
    "x = BatchNormalization()\n",
    "x = Dense(32, activation='relu')(x)\n",
    "x = BatchNormalization()\n",
    "x = Dense(16, activation='relu')(x)\n",
    "x = BatchNormalization()\n",
    "predictions = Dense(10, activation='softmax')(x)\n",
    "\n",
    "rcnn_model = Model(inputs=inputs, outputs=predictions)\n",
    "rcnn_model.compile(optimizer= 'adam',\n",
    "              loss=\"categorical_crossentropy\",\n",
    "              metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rcnn_model.fit(X_sub_train, y_sub_train, batch_size = 32, epochs = 75, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Compute and print the loss and accuracy\n",
    "\n",
    "score = rcnn_model.evaluate(X_sub_test, y_sub_test, verbose=0)\n",
    "model4_loss = score[0]\n",
    "model4_acc = score[1]\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Draw a learning curve \n",
    "\n",
    "epochs = 75\n",
    "learning_curve(rcnn_model, X_sub_train, y_sub_train, epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss and Score Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparation for creating a dataframe for all the result \n",
    "\n",
    "loss = {'loss':[model1_loss,model2_loss,model3_loss,model4_loss]}\n",
    "acc = {'test accuracy':[model1_acc,model2_acc,model3_acc,model4_acc]}\n",
    "\n",
    "loss_df = pd.DataFrame(data=loss,index = ['Vanila Model', 'Vanila Model Using Drop-out',\n",
    "                                          'Batch Normalization', 'Residual Connections'])\n",
    "acc_df = pd.DataFrame(data=acc, index = ['Vanila Model', 'Vanila Model Using Drop-out'\n",
    "                                         , 'Batch Normalization', 'Residual Connections'])\n",
    "\n",
    "df = df_concat = pd.concat([loss_df, acc_df], axis=1)\n",
    "\n",
    "# Display the dataframe\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/contaent/gdrive', force_remount=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile, os\n",
    "from scipy import misc\n",
    "import imageio\n",
    "import matplotlib.pyplot as pl\n",
    "import os\n",
    "import fnmatch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "from sklearn.model_selection import train_test_split\n",
    "import cv2\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from keras.preprocessing.image import ImageDataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "device_name = tf.test.gpu_device_name()\n",
    "if device_name != '/device:GPU:0':\n",
    "    raise SystemError('GPU device not found')\n",
    "print('Found GPU at: {}'.format(device_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path =  \"gdrive/My Drive/hw5/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting all the paths of the images\n",
    "configfiles = [os.path.join(dirpath, f)\n",
    "    for dirpath, dirnames, files in os.walk(path+\"new\")\n",
    "    for f in fnmatch.filter(files, '*.png')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub sample size\n",
    "sample_size = 5000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#show 10 random images\n",
    "for i in range(10):\n",
    "    n = int(np.random.rand() * sample_size * 2)\n",
    "    f = cv2.imread(configfiles[n])\n",
    "    print(configfiles[n])\n",
    "    print(f.shape)\n",
    "    plt.imshow(f)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort the paths into class 0 and class 1\n",
    "class0 = [i for i in configfiles if \"class0\" in i]\n",
    "class1 = [i for i in configfiles if \"class1\" in i]\n",
    "print(\"Number of sample in class 0 is {}\".format(len(class0)))\n",
    "print(\"Number of sample in class 1 is {}\".format(len(class1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_size = 5000\n",
    "\n",
    "from keras.preprocessing import image\n",
    "class0_arr = np.array([cv2.imread(class0[0])])\n",
    "count = 1\n",
    "ss = 1\n",
    "test = []\n",
    "while count < sample_size:\n",
    "    \n",
    "    if count%100 == 0:\n",
    "        print(count)\n",
    "    #current =  np.array([imageio.imread(class0[count],as_gray=True,pilmode=\"RGB\")])\n",
    "    img = image.load_img(class0[count], target_size=(50,50,3))\n",
    "    current = np.array([image.img_to_array(img)])\n",
    "    #test.append(image.img_to_array(img))\n",
    "    img.close()\n",
    "    class0_arr = np.concatenate((class0_arr,current),axis=0)\n",
    "    count += 1\n",
    "\n",
    "\n",
    "\n",
    "class0_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_size = 5000\n",
    "class1_arr = np.array([cv2.imread(class1[0])])\n",
    "count = 1\n",
    "ss = 1\n",
    "while ss < sample_size or count < sample_size:\n",
    "    if count%100 == 0:\n",
    "        print(count)\n",
    "#     current =  np.array([imageio.imread(class1[count],as_gray=True,pilmode=\"RGB\")])\n",
    "    current = np.array([cv2.imread(class1[count])])\n",
    "\n",
    "    if current.shape[1:] == (50,50,3):\n",
    "        class1_arr = np.concatenate((class1_arr,current),axis=0)\n",
    "        ss += 1\n",
    "    count += 1\n",
    "\n",
    "class1_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating balanced dataset 20k class0 and 30k class1\n",
    "y_0 = np.zeros(sample_size)\n",
    "y_1 = y_0 + 1\n",
    "y = np.append(y_0,y_1)\n",
    "\n",
    "X = np.concatenate((class0_arr,class1_arr),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X,y, stratify=y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = (50,50,3)\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "y_train_cat = to_categorical(y_train)\n",
    "y_test_cat = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten, SeparableConv2D\n",
    "\n",
    "\n",
    "cnv_bn = Sequential([\n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "           input_shape = input_shape, activation = 'relu', padding=\"same\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu',padding=\"same\"),\n",
    "    BatchNormalization(),\n",
    "    Flatten(),\n",
    "    Dense(64, activation='relu'),\n",
    "    Dense(2, activation = 'softmax')\n",
    "    \n",
    "])\n",
    "from keras.optimizers import SGD, Adam, Adagrad\n",
    "opt = SGD(lr=0.01)\n",
    "#opt = Adam(lr=0.001)\n",
    "#opt = Adagrad(lr=0.01, decay=0.01 / 10)\n",
    "cnv_bn.compile(optimizer = opt, loss=\"categorical_crossentropy\", \n",
    "                      metrics=['accuracy'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnv_bn.fit(X_train, y_train_cat, validation_split=0.2, batch_size = 30, epochs = 30, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cnv_bn.evaluate(X_test, y_test_cat, verbose=0)\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating image data generator to distort image\n",
    "generator = ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=50,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    horizontal_flip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator.fit(X_train)\n",
    "\n",
    "cnv_bn.fit_generator(generator.flow(X_train, y_train_cat, batch_size=32),\n",
    "          steps_per_epoch=X_train.shape[0]//32,\n",
    "         epochs=30,\n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cnv_bn.evaluate(X_test, y_test_cat, verbose=0)\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnv_bn = Sequential([\n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "           input_shape = input_shape, activation = 'relu', padding=\"same\"),\n",
    "    Dense(256, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(256, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(256, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    \n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu',padding=\"same\"),\n",
    "    Dense(128, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(128, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(128, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    \n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu',padding=\"same\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    \n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu',padding=\"same\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    \n",
    "    Conv2D(128, kernel_size = (3,3),\n",
    "          input_shape = input_shape, activation = 'relu',padding=\"same\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    Dense(64, input_shape=input_shape, activation=\"relu\"),\n",
    "    BatchNormalization(),\n",
    "    \n",
    "    Flatten(),\n",
    "    Dense(64, activation='relu'),\n",
    "    Dense(2, activation = 'softmax')\n",
    "    \n",
    "])\n",
    "from keras.optimizers import SGD, Adam, Adagrad\n",
    "opt = SGD(lr=0.01)\n",
    "#opt = Adam(lr=0.001)\n",
    "#opt = Adagrad(lr=0.01, decay=0.01 / 10)\n",
    "cnv_bn.compile(optimizer = opt, loss=\"categorical_crossentropy\", \n",
    "                      metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnv_bn.fit(X_train, y_train_cat, validation_split=0.2, batch_size = 30, epochs = 30, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cnv_bn.evaluate(X_test, y_test_cat, verbose=0)\n",
    "print(\"Test loss: {:.3f}\".format(score[0]))\n",
    "print(\"Test Accuracy: {:.3f}\".format(score[1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}