3DResNetModel Update 2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3DResNetModel Update #2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NX5jXq7XxGi5"
      },
      "source": [
        "# **UTMIST: 3D ResNet Model**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Li0cJjy0xCGA"
      },
      "source": [
        "## **Import Libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jirxidQg_U1"
      },
      "source": [
        "%matplotlib inline\n",
        "%config InlineBackend.figure_format = \"retina\"\n",
        "\n",
        "import torch\n",
        "import numpy as np\n",
        "from torchvision import datasets, models\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import matplotlib.pyplot as plt\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hFCRMFUmxN0_"
      },
      "source": [
        "## **Loading Data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cHS2iaAJ-C8g"
      },
      "source": [
        "#Potentially try to get kinetics dataset\n",
        "!tar xvzf https://storage.googleapis.com/deepmind-media/Datasets/kinetics400.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "od6L4XhnxP31",
        "outputId": "0daedff6-50e6-4c4e-e6eb-2a98f7ba1ec5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "source": [
        "#Temporary Kinetic-400 dataset only for testing mode. Read more here: https://pytorch.org/docs/stable/torchvision/datasets.html#kinetics-400\n",
        "# number of subproccesses to use for data loading\n",
        "\n",
        "#This code isn't working. I will try to research more into this: https://github.com/pytorch/vision/issues/1271 or this to help: https://github.com/NVIDIA/nvvl or to just see if \n",
        "\n",
        "num_workers = 0\n",
        "# how many samples per batch to load\n",
        "batch_size = 20\n",
        "# percentage of training data to be used as validation\n",
        "valid_size = 0.2\n",
        "\n",
        "# convert data to a normalized torch.FloatTensor\n",
        "transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "# choose the training and test datasets\n",
        "train_data = datasets.Kinetics400(root='data', transform=transform)\n",
        "test_data = datasets.Kinetics400(root='data', transform=transform)\n",
        "\n",
        "# obtain training indices that will be used for validation\n",
        "num_train = len(train_data)\n",
        "indices = list(range(num_train))\n",
        "np.random.shuffle(indices)\n",
        "split = int(np.floor(valid_size * num_train))\n",
        "train_idx, valid_idx = indices[split:], indices[:split]\n",
        "\n",
        "# define samplers for obtaining training and validation batches\n",
        "train_sampler = SubsetRandomSampler(train_idx)\n",
        "valid_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "# prepare data loaders(combine dataset and sampler)\n",
        "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\n",
        "valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\n",
        "test_loader = torch.utils.data.DataLoader(test_data, batch_size, num_workers=num_workers)\n",
        "\n",
        "\n",
        "# print out some data stats\n",
        "print('Num training images: ', len(train_data))\n",
        "print('Num test images: ', len(test_data))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-51fcc3a1b476>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# choose the training and test datasets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mtrain_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mKinetics400\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0mtest_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mKinetics400\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: __init__() missing 1 required positional argument: 'frames_per_clip'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g-klDTwHxQSZ"
      },
      "source": [
        "## **Visualize A Batch Of Training Data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O4awCQjZxUJy"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z1UOAYJgxUoH"
      },
      "source": [
        "## **Checking If GPU Is Available**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TAFXX4etxawJ"
      },
      "source": [
        "#Use GPU if it's available, may not work with Azure\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "print(\"The ML code in this project will be trained on \" + str(device))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rg1g-DeoxbTL"
      },
      "source": [
        "## **Defining The ML Model**\n",
        "\n",
        "This is done as per section **3.1 Network Architecture** in the [paper](https://arxiv.org/pdf/1708.07632.pdf) we are reproducing."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I8obE_Jz5QL-"
      },
      "source": [
        "### **3D ResNet-18**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iglzAQeKlrzp"
      },
      "source": [
        "class Net(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        \n",
        "        self.conv_1 = nn.Sequential(nn.Conv3d(in_channels=3, out_channels=64, kernel_size=7, stride=1, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=64),\n",
        "                                    nn.ReLU())\n",
        "        \n",
        "        self.maxpool = nn.MaxPool3d(kernel_size=3, stride=2, padding=1) #swap w/ avg pool if this doesn't work\n",
        "        \n",
        "\n",
        "        self.conv_2 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=64, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=64),\n",
        "                                    nn.ReLU())\n",
        "\n",
        "        self.conv_3 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU())\n",
        "        \n",
        "        self.conv_3_1 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=64, out_channels=128 stride=2,), \n",
        "                                    nn.BatchNorm3d(num_features=128))\n",
        "\n",
        "        self.conv_4 = nn.Sequential(nn.Conv3d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=256),\n",
        "                                    nn.ReLU())    \n",
        "\n",
        "        self.conv_4_1 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=128, out_channels=256, stride=2), \n",
        "                                    nn.BatchNorm3d(num_features=256))   \n",
        "\n",
        "        self.conv_5 = nn.Sequential(nn.Conv3d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=512),\n",
        "                                    nn.ReLU())\n",
        "\n",
        "        self.conv_5_1 = nn.Sequential(nn.Conv3d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=512),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=256, out_channels=512, stride=2), \n",
        "                                    nn.BatchNorm3d(num_features=512))                                            \n",
        "                                      \n",
        "        self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1))\n",
        "\n",
        "        self.fc = nn.Linear(in_features=512, out_features=400)\n",
        "\n",
        "        self.downsample = nn.Sequential(conv1x1x1(in_channels=in_channels, out_channels=out_channels, stride=2), nn.BatchNorm3d(planes * block.expansion))  \n",
        "      \n",
        "        \n",
        "    def forward(self, x):\n",
        "        # flatten image input\n",
        "        x = x.view(x.size(0), -1)\n",
        "        #conv_1      \n",
        "        x = self.conv_1(x)\n",
        "        #conv_2 \n",
        "        x = self.conv_2(x)\n",
        "        x = self.conv_2(x)\n",
        "        #conv_3\n",
        "        x = self.conv_3_1(x)\n",
        "        x = self.conv_3(x)\n",
        "        #conv_4\n",
        "        x = self.conv_4_1(x)\n",
        "        x = self.conv_4(x)\n",
        "        #conv_5\n",
        "        x = self.conv_5_1(x)\n",
        "        x = self.conv_5(x)\n",
        "        #avgpool\n",
        "        x = self.avgpool(x)\n",
        "        #fully_connected_layer\n",
        "        x = self.fc(x)\n",
        "        x = F.softmax(dim=3)\n",
        "        return x\n",
        "\n",
        "\n",
        "# initialize the CNN\n",
        "model = Net()\n",
        "               \n",
        "#Defining the loss\n",
        "criterion = nn.softmax()\n",
        "\n",
        "# Defining the optimizer\n",
        "optimizer = optim.SGD(model, lr=0.001)\n",
        "\n",
        "#Sending model to CPU/GPU, may not work with Azure (check)\n",
        "model.to(device);\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oVQnDcFf5T0K"
      },
      "source": [
        "### **3d ResNet-34**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLTyR3IK5XYq"
      },
      "source": [
        "class Net34(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        \n",
        "        self.conv_1 = nn.Sequential(nn.Conv3d(in_channels=3, out_channels=64, kernel_size=7, stride=1, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=64),\n",
        "                                    nn.ReLU())\n",
        "        \n",
        "        self.maxpool = nn.MaxPool3d((3, 3, 3)) #swap w/ avg pool if this doesn't work\n",
        "\n",
        "        self.conv_2 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=64, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=64),\n",
        "                                    nn.ReLU())\n",
        "\n",
        "        self.conv_3 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU())\n",
        "        \n",
        "        self.conv_3_1 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=64, out_channels=128 stride=2,), \n",
        "                                    nn.BatchNorm3d(num_features=128))\n",
        "\n",
        "        self.conv_4 = nn.Sequential(nn.Conv3d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=256),\n",
        "                                    nn.ReLU())    \n",
        "\n",
        "        self.conv_4_1 = nn.Sequential(nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=128),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=128, out_channels=256, stride=2), \n",
        "                                    nn.BatchNorm3d(num_features=256))   \n",
        "\n",
        "        self.conv_5 = nn.Sequential(nn.Conv3d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=512),\n",
        "                                    nn.ReLU())\n",
        "\n",
        "        self.conv_5_1 = nn.Sequential(nn.Conv3d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1, bias=False),\n",
        "                                    nn.BatchNorm3d(num_features=512),\n",
        "                                    nn.ReLU(),\n",
        "                                    conv1x1x1(in_channels=256, out_channels=512, stride=2), \n",
        "                                    nn.BatchNorm3d(num_features=512))                                            \n",
        "                                      \n",
        "        self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1))\n",
        "\n",
        "        self.fc = nn.Linear(in_features=512, out_features=400)\n",
        "\n",
        "        self.downsample = nn.Sequential(conv1x1x1(in_channels=in_channels, out_channels=out_channels, stride=2), nn.BatchNorm3d(planes * block.expansion))  \n",
        "      \n",
        "        \n",
        "    def forward(self, x):\n",
        "        # flatten image input\n",
        "        x = x.view(x.size(0), -1)\n",
        "        #conv_1      \n",
        "        x = self.conv_1(x)\n",
        "        #conv_2 \n",
        "        x = self.conv_2(x)\n",
        "        x = self.conv_2(x)\n",
        "        x = self.conv_2(x)\n",
        "        #conv_3\n",
        "        x = conv_3_1\n",
        "        x = x.conv_3\n",
        "        x = x.conv_3\n",
        "        x = x.conv_3\n",
        "        #conv_4\n",
        "        x = x.conv_4_1\n",
        "        x = x.conv_4\n",
        "        x = x.conv_4\n",
        "        x = x.conv_4\n",
        "        x = x.conv_4\n",
        "        x = x.conv_4\n",
        "        #conv_5\n",
        "        x = x.conv_5_1\n",
        "        x = x.conv_5\n",
        "        x = x.conv_5\n",
        "        #avgpool\n",
        "        x = self.avgpool(x)\n",
        "        #fully_connected_layer\n",
        "        x = self.fc(x)\n",
        "        x = F.softmax(dim=3)\n",
        "        return x\n",
        "\n",
        "\n",
        "# TODO: Initialize Model, Criterion, Optimizer\n",
        "\n",
        "# initialize the CNN\n",
        "model = Net34()\n",
        "               \n",
        "#Defining the loss\n",
        "criterion = nn.softmax()\n",
        "\n",
        "# Defining the optimizer\n",
        "optimizer = optim.SGD(model, lr=0.001)\n",
        "\n",
        "#Sending model to CPU/GPU\n",
        "model.to(device);\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "liDrutWxgFcZ"
      },
      "source": [
        "## **Training loop**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1FuRzoTKl9Gq"
      },
      "source": [
        "# TODO: Training Loop, azure machine learning distributive?\n",
        "\n",
        "n_epochs = 1\n",
        "epochs_no_improve = 0\n",
        "n_epochs_stop = 5\n",
        "\n",
        "# initialize tracker for minimum validation loss\n",
        "valid_loss_min = np.Inf # set initial \"min\" to infinity\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "    # monitor training loss\n",
        "    train_loss = 0.0\n",
        "    valid_loss = 0.0\n",
        "    \n",
        "    ###################\n",
        "    # train the model #\n",
        "    ###################\n",
        "    model.train() # prep model for training\n",
        "    for i, (data, target) in train_loader:\n",
        "        # move data and target tensors to the default device\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        # forward pass: compute predicted outputs by passing inputs to the model\n",
        "        output = model(data)\n",
        "        # calculate the loss\n",
        "        loss = criterion(output, target)\n",
        "        # clear the gradients of all optimized variables\n",
        "        optimizer.zero_grad()\n",
        "        # backward pass: compute gradient of the loss with respect to model parameters\n",
        "        loss.backward()\n",
        "        # perform a single optimization step (parameter update)\n",
        "        optimizer.step()\n",
        "        # update running training loss\n",
        "        train_loss += loss.item()*data.size(0)\n",
        "        \n",
        "    ######################    \n",
        "    # validate the model #\n",
        "    ######################\n",
        "    model.eval() # prep model for evaluation\n",
        "    for data, target in valid_loader:\n",
        "        # move data and target tensors to the default device\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        # forward pass: compute predicted outputs by passing inputs to the model\n",
        "        output = model(data)\n",
        "        # calculate the loss\n",
        "        loss = criterion(output, target)\n",
        "        # update running validation loss \n",
        "        valid_loss += loss.item()*data.size(0)\n",
        "        \n",
        "    # print training/validation statistics \n",
        "    # calculate average loss over an epoch\n",
        "    train_loss = train_loss/len(train_loader.sampler)\n",
        "    valid_loss = valid_loss/len(valid_loader.sampler)\n",
        "    \n",
        "    print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n",
        "        epoch+1, \n",
        "        train_loss,\n",
        "        valid_loss\n",
        "        ))\n",
        "    \n",
        "    # save model if validation loss has decreased\n",
        "    if valid_loss <= valid_loss_min:\n",
        "        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "        valid_loss_min,\n",
        "        valid_loss))\n",
        "        torch.save(model.state_dict(), 'model_cifar.pt')\n",
        "        valid_loss_min = valid_loss\n",
        "        epochs_no_improve = 0\n",
        "    \n",
        "    else:\n",
        "        epochs_no_improve += 1\n",
        "        # Check early stopping condition\n",
        "        if epochs_no_improve == n_epochs_stop:\n",
        "            print('Early stopping!')\n",
        "            break"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}