09_Deep_NN_MNIST_CUDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 09_Deep_NN_MNIST_CUDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Warnning] This section required GPU device."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:39:41.364218Z",
     "start_time": "2017-08-04T06:39:41.345636Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fc9f5dbe390>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pytorch Library\n",
    "import torch\n",
    "import torch.nn.init\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import torchvision.utils as utils\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "torch.manual_seed(777)  # reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:39:42.231086Z",
     "start_time": "2017-08-04T06:39:42.177995Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MNIST dataset\n",
    "mnist_train = dsets.MNIST(root='data/',\n",
    "                          train=True,\n",
    "                          transform=transforms.ToTensor(),\n",
    "                          download=True)\n",
    "\n",
    "mnist_test = dsets.MNIST(root='data/',\n",
    "                         train=False,\n",
    "                         transform=transforms.ToTensor(),\n",
    "                         download=True)\n",
    "\n",
    "# Hyper-parameters\n",
    "batch_size = 100\n",
    "\n",
    "# dataset loader\n",
    "data_loader = torch.utils.data.DataLoader(dataset=mnist_train,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=True,\n",
    "                                          num_workers=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep Neural Network Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:47:58.535965Z",
     "start_time": "2017-08-04T06:47:58.435212Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential (\n",
      "  (0): Linear (784 -> 512)\n",
      "  (1): ReLU ()\n",
      "  (2): Linear (512 -> 512)\n",
      "  (3): ReLU ()\n",
      "  (4): Linear (512 -> 512)\n",
      "  (5): ReLU ()\n",
      "  (6): Linear (512 -> 512)\n",
      "  (7): ReLU ()\n",
      "  (8): Linear (512 -> 256)\n",
      "  (9): ReLU ()\n",
      "  (10): Linear (256 -> 128)\n",
      "  (11): ReLU ()\n",
      "  (12): Linear (128 -> 10)\n",
      "  (13): ReLU ()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Deep Neural Network\n",
    "linear1 = torch.nn.Linear(784, 512, bias=True)\n",
    "linear2 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear3 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear4 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear5 = torch.nn.Linear(512, 256, bias=True)\n",
    "linear6 = torch.nn.Linear(256, 128, bias=True)\n",
    "linear7 = torch.nn.Linear(128, 10, bias=True)\n",
    "relu = torch.nn.ReLU()\n",
    "#sigmoid = torch.nn.Sigmoid()\n",
    "\n",
    "# xavier initializer\n",
    "# torch.nn.init.xavier_uniform(linear1.weight)\n",
    "# torch.nn.init.xavier_uniform(linear2.weight)\n",
    "# torch.nn.init.xavier_uniform(linear3.weight)\n",
    "# torch.nn.init.xavier_uniform(linear4.weight)\n",
    "# torch.nn.init.xavier_uniform(linear5.weight)\n",
    "# torch.nn.init.xavier_uniform(linear6.weight)\n",
    "# torch.nn.init.xavier_uniform(linear7.weight)\n",
    "\n",
    "# model\n",
    "model = torch.nn.Sequential(linear1, relu,\n",
    "                            linear2, relu,\n",
    "                            linear3, relu,\n",
    "                            linear4, relu,\n",
    "                            linear5, relu,\n",
    "                            linear6, relu,\n",
    "                            linear7, relu)   \n",
    "model.cuda()\n",
    "\n",
    "#model.load_state_dict(torch.load('DNN.pkl'))  # Load the Trained Model\n",
    "\n",
    "print(model)\n",
    "# print('Weight matrix: ', model.weight.data)\n",
    "# print('bias vector: ', model.bias.data)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:50:49.107404Z",
     "start_time": "2017-08-04T06:48:03.383303Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch:    1] cost = 0.556651652\n",
      "[Epoch:    2] cost = 0.343746841\n",
      "[Epoch:    3] cost = 0.311643869\n",
      "[Epoch:    4] cost = 0.291136444\n",
      "[Epoch:    5] cost = 0.283704191\n",
      "[Epoch:    6] cost = 0.272350699\n",
      "[Epoch:    7] cost = 0.268389672\n",
      "[Epoch:    8] cost = 0.262971342\n",
      "[Epoch:    9] cost = 0.265035629\n",
      "[Epoch:   10] cost = 0.255123436\n",
      "[Epoch:   11] cost = 0.251195043\n",
      "[Epoch:   12] cost = 0.251167089\n",
      "[Epoch:   13] cost = 0.24736838\n",
      "[Epoch:   14] cost = 0.251571566\n",
      "[Epoch:   15] cost = 0.246433824\n",
      "[Epoch:   16] cost = 0.245539501\n",
      "[Epoch:   17] cost = 0.243946388\n",
      "[Epoch:   18] cost = 0.243684247\n",
      "[Epoch:   19] cost = 0.24546963\n",
      "[Epoch:   20] cost = 0.241581306\n",
      "[Epoch:   21] cost = 0.241334602\n",
      "[Epoch:   22] cost = 0.239114419\n",
      "[Epoch:   23] cost = 0.243339986\n",
      "[Epoch:   24] cost = 0.241198525\n",
      "[Epoch:   25] cost = 0.237277612\n",
      "[Epoch:   26] cost = 0.241510257\n",
      "[Epoch:   27] cost = 0.239520103\n",
      "[Epoch:   28] cost = 0.238840908\n",
      "[Epoch:   29] cost = 0.23869592\n",
      "[Epoch:   30] cost = 0.23714681\n",
      "Learning Finished!\n"
     ]
    }
   ],
   "source": [
    "# Softmax 함수가 Cost를 계산할 때 내장되어 있다.\n",
    "cost_func = torch.nn.CrossEntropyLoss()   \n",
    "\n",
    "# Hyper-parameters\n",
    "learning_rate = 0.001 \n",
    "training_epochs = 30\n",
    "\n",
    "# Adam Optimizer\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# Train model\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0\n",
    "    total_batch = len(mnist_train) // batch_size\n",
    "\n",
    "    for i, (batch_images, batch_labels) in enumerate(data_loader):\n",
    "        \n",
    "        # 이미지를 [batch_size x 784] size 행렬로 변환\n",
    "        \n",
    "        X = Variable(batch_images.view(-1, 28 * 28)).cuda()\n",
    "        Y = Variable(batch_labels).cuda()  # label is not one-hot encoded\n",
    "\n",
    "        optimizer.zero_grad()             # Zero Gradient Container\n",
    "        Y_prediction = model(X)           # Forward Propagation\n",
    "        cost = cost_func(Y_prediction, Y) # compute cost\n",
    "        cost.backward()                   # compute gradient\n",
    "        optimizer.step()                  # gradient update\n",
    "\n",
    "        avg_cost += cost / total_batch\n",
    "\n",
    "    print(\"[Epoch: {:>4}] cost = {:>.9}\".format(epoch + 1, avg_cost.data[0]))\n",
    "\n",
    "print('Learning Finished!')\n",
    "\n",
    "# torch.save(model.state_dict(), 'DNN.pkl')  # Save the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Set을 이용한 모형 성능 측정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:57:57.681213Z",
     "start_time": "2017-08-04T06:57:45.301222Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 88 %\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('DNN.pkl'))\n",
    "\n",
    "# Test the Model\n",
    "correct = 0\n",
    "total = 0\n",
    "for images, labels in mnist_test:\n",
    "    images = Variable(images.view(-1, 28*28)).cuda()\n",
    "    outputs = model(images)\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "    total += 1\n",
    "    correct += (predicted == labels).sum()\n",
    "\n",
    "print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Sample Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:57:58.752269Z",
     "start_time": "2017-08-04T06:57:58.458742Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label:  \n",
      " 2\n",
      "[torch.cuda.LongTensor of size 1 (GPU 0)]\n",
      "\n",
      "Prediction:  \n",
      " 2\n",
      "[torch.cuda.LongTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADjxJREFUeJzt3X+MVfWZx/HPIz+iTvljkEgG6zK1aiPRRNaJ9g+yYro2\nYohYiYp/zWbrTmMwbBMlEqtZjdmk2Vg2arSEpqTDpmshEQNpjC0lplZjqoCsImyRrUMYMjISNAUF\nuzDP/jFnNgPO/Z7Lvefcc5nn/Uomc+957jn38Q4fzz33e8/5mrsLQDwXVN0AgGoQfiAowg8ERfiB\noAg/EBThB4Ii/EBQhB8IivADQU1t5ZOZGV8nBErm7lbP45ra85vZbWb2JzPbb2armtkWgNayRr/b\nb2ZTJO2TdKukQUnvSLrP3fck1mHPD5SsFXv+GyXtd/c/u/tfJf1K0pImtgeghZoJ/2WSDo67P5gt\nO4OZ9ZnZdjPb3sRzAShY6R/4uftaSWsl3vYD7aSZPf8hSZePu//1bBmA80Az4X9H0lVm9g0zmy5p\nmaQtxbQFoGwNv+1391Nm9qCk30iaImmdu39QWGcAStXwUF9DT8YxP1C6lnzJB8D5i/ADQRF+ICjC\nDwRF+IGgCD8QVEvP5wfG6+joSNYXL16crM+dOzdZf+aZZ2rWvvzyy+S6EbDnB4Ii/EBQhB8IivAD\nQRF+ICjCDwTFUB8q8/jjjyfrK1euTNbzzkh99913a9a2bt2aXDcC9vxAUIQfCIrwA0ERfiAowg8E\nRfiBoAg/EBTj/CjV/fffX7O2YsWKFnaCs7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgmpql18wG\nJB2TdFrSKXfvyXk8s/SeZzo7O5P15cuXJ+uPPPJIzdpFF12UXNcsPdls3r/djz76qGbthRdeSK67\nZs2aZP3EiRPJepXqnaW3iC/53OLuRwrYDoAW4m0/EFSz4XdJvzWzHWbWV0RDAFqj2bf9C9z9kJld\nKmmrmf23u78+/gHZ/xT4HwPQZpra87v7oez3sKSXJd04wWPWuntP3oeBAFqr4fCbWYeZzRi7Lem7\nknYX1RiAcjXztn+2pJez4Zipkv7T3V8tpCsApWtqnP+cn4xx/razaNGiZL2/vz9ZnzlzZpHtnCFv\nLH1gYCBZv+aaaxp+7jlz5iTrw8PDDW+7bPWO8zPUBwRF+IGgCD8QFOEHgiL8QFCEHwiKob5JbuHC\nhcn6a6+9lqyPjIw09fwnT56sWdu2bVty3TvuuCNZ7+3tTdZTp+VOnz49ue7TTz+drKdOVa4aQ30A\nkgg/EBThB4Ii/EBQhB8IivADQRF+ICim6J7kenrSF1AaHBxsavuffPJJsv7kk0/WrG3evLmp5964\ncWOy3tdX++pxN910U3LdI0cm/wWp2fMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCczz/JdXd3l7r9\nvMtnlynvfP9NmzbVrA0NDSXXve6665L1zz77LFmvEufzA0gi/EBQhB8IivADQRF+ICjCDwRF+IGg\ncs/nN7N1khZLGnb3a7NlMyVtkNQtaUDSPe7+aXltolFVjsOXbenSpQ2vmzdnQDuP4xelnj3/LyTd\ndtayVZK2uftVkrZl9wGcR3LD7+6vSzp61uIlkvqz2/2S7iy4LwAla/SYf7a7j30/8mNJswvqB0CL\nNH0NP3f31Hf2zaxPUu2LqQGoRKN7/sNm1iVJ2e/hWg9097Xu3uPu6StJAmipRsO/RdLYFKm9kpq7\nDCuAlssNv5m9KOktSd8ys0Ez+76kH0u61cw+lPT32X0A55HcY353v69G6TsF9wKc4dJLL03W77rr\nrhZ1MjnxDT8gKMIPBEX4gaAIPxAU4QeCIvxAUEzRjbZ1yy23JOsXX3xxw9tevXp1w+tOFuz5gaAI\nPxAU4QeCIvxAUIQfCIrwA0ERfiAoxvnRth577LFkPW96+d27d9esHTx4sKGeJhP2/EBQhB8IivAD\nQRF+ICjCDwRF+IGgCD8QFOP8qEx3d3eyPm/evGQ9b5z/rbfeqln79FNmlGfPDwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANB5Y7zm9k6SYslDbv7tdmyJyT9k6RPsoc96u6vlNVku7v77ruT9QsvvDBZv+GG\nG5L1HTt2nHNPY/bs2VPatuuxePHimrVmr53/5ptvJuvPP/98U9uf7OrZ8/9C0m0TLP93d78++wkb\nfOB8lRt+d39d0tEW9AKghZo55n/QzN4zs3Vm1llYRwBaotHw/1TSNyVdL2lI0k9qPdDM+sxsu5lt\nb/C5AJSgofC7+2F3P+3uI5J+JunGxGPXunuPu/c02iSA4jUUfjPrGnf3e5JqXyYVQFuqZ6jvRUkL\nJc0ys0FJ/yJpoZldL8klDUj6QYk9AiiB5Z0TXeiTmZX2ZB0dHcn6Aw88kKwvXbo0WZ87d27N2qxZ\ns5LrTpkyJVkv82/wxRdfJOuff/55sv7KK+lR3Lxz7ufPn1+zNnVqet9z4sSJZP3mm29O1nfu3Jms\nT1bubvU8jm/4AUERfiAowg8ERfiBoAg/EBThB4KaNEN9b7/9drKed9ps3qWcd+3aVbP26quvJtd9\n6KGHkvW84bS9e/cm6/39/TVrzz33XHLde++9N1kfGRlJ1st0wQXpfVMzveWd6vzUU08l6xs3bmz4\nucvGUB+AJMIPBEX4gaAIPxAU4QeCIvxAUIQfCGrSjPOfPn06Wc/771y0aFGyvnXr1nPuqSgzZsxI\n1q+44oqatWeffTa57oIFC5L1Zv997Nu3r2bt5MmTyXXnzJmTrOedSl2mvNORq8Q4P4Akwg8ERfiB\noAg/EBThB4Ii/EBQhB8IatKM82/YsCFZz7s0d95lno8erT1X6Zo1a5Lr5l02PO9vcMkllyTrqctj\n5zFLDwkfO3YsWX/44YeT9dTfJW/bV199dbKeupy6lP6bL1u2LLlu3t901apVyXqVGOcHkET4gaAI\nPxAU4QeCIvxAUIQfCIrwA0HljvOb2eWS1kuaLcklrXX3Z8xspqQNkrolDUi6x92TF78vc5y/q6sr\nWV+xYkWyvnLlyiLbOUPeWHqZ37XIuz79G2+8kayvXr06Wd+/f/8599QOrrzyymR9YGAgWT916lSB\n3RSryHH+U5Iecvd5kr4tabmZzZO0StI2d79K0rbsPoDzRG743X3I3Xdmt49J2ivpMklLJI1NFdMv\n6c6ymgRQvHM65jezbknzJf1R0mx3H8pKH2v0sADAeaLuC5GZ2dckvSTph+7+l/HHse7utY7nzaxP\nUl+zjQIoVl17fjObptHg/9LdN2WLD5tZV1bvkjQ80bruvtbde9y9p4iGARQjN/w2uov/uaS97j7+\no98tknqz272SNhffHoCy1DPUt0DSHyS9L2lsTuRHNXrcv1HS30g6oNGhvtrnvarcob4806ZNS9Y7\nOzuT9d7e3pq1Ki8hLUnr16+vWTtw4EBy3ePHjxfdDipW71Bf7jG/u78hqdbGvnMuTQFoH3zDDwiK\n8ANBEX4gKMIPBEX4gaAIPxDUpLl0N4BRXLobQBLhB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ElRt+M7vc\nzF4zsz1m9oGZ/XO2/AkzO2Rmu7Kf28tvF0BRciftMLMuSV3uvtPMZkjaIelOSfdIOu7uT9f9ZEza\nAZSu3kk7ptaxoSFJQ9ntY2a2V9JlzbUHoGrndMxvZt2S5kv6Y7boQTN7z8zWmVlnjXX6zGy7mW1v\nqlMAhap7rj4z+5qk30v6V3ffZGazJR2R5JKe0uihwT/mbIO3/UDJ6n3bX1f4zWyapF9L+o27r56g\n3i3p1+5+bc52CD9QssIm6jQzk/RzSXvHBz/7IHDM9yTtPtcmAVSnnk/7F0j6g6T3JY1kix+VdJ+k\n6zX6tn9A0g+yDwdT22LPD5Ss0Lf9RSH8QPkKe9sPYHIi/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBJV7Ac+CHZF0YNz9WdmydtSuvbVrXxK9NarI3ubW+8CWns//\nlSc32+7uPZU1kNCuvbVrXxK9Naqq3njbDwRF+IGgqg7/2oqfP6Vde2vXviR6a1QlvVV6zA+gOlXv\n+QFUpJLwm9ltZvYnM9tvZquq6KEWMxsws/ezmYcrnWIsmwZt2Mx2j1s208y2mtmH2e8Jp0mrqLe2\nmLk5MbN0pa9du8143fK3/WY2RdI+SbdKGpT0jqT73H1PSxupwcwGJPW4e+Vjwmb2d5KOS1o/NhuS\nmf2bpKPu/uPsf5yd7v5Im/T2hM5x5uaSeqs1s/Q/qMLXrsgZr4tQxZ7/Rkn73f3P7v5XSb+StKSC\nPtqeu78u6ehZi5dI6s9u92v0H0/L1eitLbj7kLvvzG4fkzQ2s3Slr12ir0pUEf7LJB0cd39Q7TXl\nt0v6rZntMLO+qpuZwOxxMyN9LGl2lc1MIHfm5lY6a2bptnntGpnxumh84PdVC9z9byUtkrQ8e3vb\nlnz0mK2dhmt+KumbGp3GbUjST6psJptZ+iVJP3T3v4yvVfnaTdBXJa9bFeE/JOnycfe/ni1rC+5+\nKPs9LOlljR6mtJPDY5OkZr+HK+7n/7n7YXc/7e4jkn6mCl+7bGbplyT90t03ZYsrf+0m6quq162K\n8L8j6Soz+4aZTZe0TNKWCvr4CjPryD6IkZl1SPqu2m/24S2SerPbvZI2V9jLGdpl5uZaM0ur4teu\n7Wa8dveW/0i6XaOf+P+PpB9V0UONvq6Q9F/ZzwdV9ybpRY2+DfxfjX428n1Jl0jaJulDSb+TNLON\nevsPjc7m/J5Gg9ZVUW8LNPqW/j1Ju7Kf26t+7RJ9VfK68Q0/ICg+8AOCIvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoAg/ENT/AWMav8COCEW1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fca15ca7eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get one and predict\n",
    "r = random.randint(0, len(mnist_test) - 1)\n",
    "X_single_data = Variable(mnist_test.test_data[r:r + 1].view(-1, 28 * 28).float()).cuda()\n",
    "Y_single_data = Variable(mnist_test.test_labels[r:r + 1]).cuda()\n",
    "\n",
    "single_prediction = model(X_single_data)\n",
    "\n",
    "print(\"Label: \", Y_single_data.data)\n",
    "print(\"Prediction: \", torch.max(single_prediction.data, 1)[1])\n",
    "\n",
    "plt.imshow(X_single_data.cpu().data.view(28,28).numpy() , cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}