mxnet-mnist_deep_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<h1> AWS Summit 2017 - Seoul: MXNet MNIST CNN Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import time\n",
    "import mxnet as mx\n",
    "import mxnet.ndarray as nd\n",
    "import numpy as np\n",
    "import random\n",
    "from sklearn.datasets import fetch_mldata\n",
    "\n",
    "# set the seeds. However, this does not guarantee that the result will always be the same since CUDNN is non-deterministic\n",
    "np.random.seed(777)\n",
    "mx.random.seed(77)\n",
    "random.seed(7777)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 1. Load the data </h3>\n",
    "\n",
    "Load the MNIST dataset. We use 55000 images for training, 5000 images for validation and 10000 images for testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = fetch_mldata(dataname='MNIST original')\n",
    "X, y = mnist.data, mnist.target\n",
    "X = X.astype(np.float32) / 255.0\n",
    "X_train, X_valid, X_test = X[:55000].reshape((-1, 1, 28, 28)),\\\n",
    "                           X[55000:60000].reshape((-1, 1, 28, 28)),\\\n",
    "                           X[60000:].reshape((-1, 1, 28, 28))\n",
    "y_train, y_valid, y_test = y[:55000], y[55000:60000], y[60000:]\n",
    "\n",
    "# hyper parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 15\n",
    "batch_size = 100\n",
    "drop_out_prob = 0.3  # The keep probability is 0.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 2. Build the symbol </h3>\n",
    "\n",
    "Next we will build the symbol, which is used to determine the data flow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = mx.sym.var(name=\"data\")\n",
    "label = mx.sym.var(name=\"label\")\n",
    "\n",
    "L1 = mx.sym.Convolution(data=data, kernel=(3, 3), pad=(1, 1), num_filter=32, name='L1_conv')\n",
    "L1 = mx.sym.Activation(data=L1, act_type='relu', name='L1_relu')\n",
    "L1 = mx.sym.Pooling(data=L1, kernel=(2, 2), stride=(2, 2), pool_type='max', name='L1_pool')\n",
    "L1 = mx.sym.Dropout(L1, p=drop_out_prob, name=\"L1_dropout\")\n",
    "\n",
    "L2 = mx.sym.Convolution(data=L1, kernel=(3, 3), pad=(1, 1), num_filter=64, name='L2_conv')\n",
    "L2 = mx.sym.Activation(data=L2, act_type='relu', name='L2_relu')\n",
    "L2 = mx.sym.Pooling(data=L2, kernel=(2, 2), stride=(2, 2), pool_type='max', name='L2_pool')\n",
    "L2 = mx.sym.Dropout(L2, p=drop_out_prob, name=\"L2_dropout\")\n",
    "\n",
    "L3 = mx.sym.Convolution(data=L2, kernel=(3, 3), pad=(1, 1), num_filter=128, name='L3_conv')\n",
    "L3 = mx.sym.Activation(data=L3, act_type='relu', name='L3_relu')\n",
    "L3 = mx.sym.Pooling(data=L3, kernel=(2, 2), stride=(2, 2), pad=(1, 1), pool_type='max', name='L3_pool')\n",
    "L3 = mx.sym.flatten(L3)\n",
    "L3 = mx.sym.Dropout(L3, p=drop_out_prob, name=\"L3_dropout\")\n",
    "\n",
    "L4 = mx.sym.FullyConnected(data=L3, num_hidden=625, name='L4_fc')\n",
    "L4 = mx.sym.Activation(data=L4, act_type='relu', name='L4_relu')\n",
    "L4 = mx.sym.Dropout(L4, p=drop_out_prob)\n",
    "\n",
    "logits = mx.sym.FullyConnected(data=L4, num_hidden=10, name='logits')\n",
    "\n",
    "loss = mx.sym.mean(-mx.sym.pick(mx.sym.log_softmax(logits), label, axis=-1))\n",
    "loss = mx.sym.make_loss(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 3. Construct the Module </h3>\n",
    "\n",
    "We will construct the Module object based on the symbol. Module will be used for training and testing. Also, the testing executor will try to reuse the allocated memory space of the training executor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_desc = mx.io.DataDesc(name='data', shape=(batch_size, 1, 28, 28), layout='NCHW')\n",
    "label_desc = mx.io.DataDesc(name='label', shape=(batch_size, ), layout='N')\n",
    "net = mx.mod.Module(symbol=loss,\n",
    "                    data_names=[data_desc.name],\n",
    "                    label_names=[label_desc.name],\n",
    "                    context=mx.gpu())\n",
    "net.bind(data_shapes=[data_desc], label_shapes=[label_desc])\n",
    "net.init_params(initializer=mx.init.Xavier())\n",
    "net.init_optimizer(optimizer=\"adam\",\n",
    "                   optimizer_params={'learning_rate': learning_rate,\n",
    "                                     'rescale_grad': 1.0},\n",
    "                   kvstore=None)\n",
    "\n",
    "# We build another testing network that outputs the logits.\n",
    "test_net = mx.mod.Module(symbol=logits,\n",
    "                         data_names=[data_desc.name],\n",
    "                         label_names=None,\n",
    "                         context=mx.gpu())\n",
    "# Setting the `shared_module` to ensure that the test network shares the same parameters and\n",
    "#  allocated memory of the training network\n",
    "test_net.bind(data_shapes=[data_desc],\n",
    "              label_shapes=None,\n",
    "              for_training=False,\n",
    "              grad_req='null',\n",
    "              shared_module=net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 4. Training </h3>\n",
    "\n",
    "We can fit the training set now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost = 0.238899125 time spent=3.11269s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0002 cost = 0.071778030 time spent=3.00522s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0003 cost = 0.055199430 time spent=3.01519s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0004 cost = 0.044752500 time spent=2.98161s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0005 cost = 0.040929539 time spent=3.02003s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0006 cost = 0.034377882 time spent=2.99162s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0007 cost = 0.032306075 time spent=3.06967s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0008 cost = 0.027965825 time spent=3.01163s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0009 cost = 0.027749191 time spent=3.05169s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0010 cost = 0.026517156 time spent=3.01613s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0011 cost = 0.024458728 time spent=3.02264s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0012 cost = 0.022148437 time spent=3.03465s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0013 cost = 0.022070018 time spent=3.00563s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0014 cost = 0.022399242 time spent=3.03108s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0015 cost = 0.019536681 time spent=3.04515s\nLearning Finished!\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(training_epochs):\n",
    "    begin = time.time()\n",
    "    avg_cost = 0\n",
    "    total_batch = int(math.ceil(X_train.shape[0] / batch_size))\n",
    "    shuffle_ind = np.random.permutation(np.arange(X_train.shape[0]))\n",
    "    X_train = X_train[shuffle_ind, :]\n",
    "    y_train = y_train[shuffle_ind]\n",
    "    for i in range(total_batch):\n",
    "        # Slice the data batch and label batch.\n",
    "        # Note that we use np.take to ensure that the batch will be padded correctly.\n",
    "        data_npy = np.take(X_train,\n",
    "                           indices=np.arange(i * batch_size, (i+1) * batch_size),\n",
    "                           axis=0,\n",
    "                           mode=\"clip\")\n",
    "        label_npy = np.take(y_train,\n",
    "                            indices=np.arange(i * batch_size, (i + 1) * batch_size),\n",
    "                            axis=0,\n",
    "                            mode=\"clip\")\n",
    "        net.forward(data_batch=mx.io.DataBatch(data=[nd.array(data_npy)],\n",
    "                                               label=[nd.array(label_npy)]),\n",
    "                    is_train=True)\n",
    "        loss_nd = net.get_outputs()[0]\n",
    "        net.backward()\n",
    "        net.update()\n",
    "        avg_cost += loss_nd.asnumpy()[0] / total_batch\n",
    "    end = time.time()\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost), 'time spent=%gs' %(end-begin))\n",
    "print('Learning Finished!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 5. Testing </h3>\n",
    "\n",
    "Let's test the model on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9924\n"
     ]
    }
   ],
   "source": [
    "total_batch = int(np.ceil(X_test.shape[0] / batch_size))\n",
    "correct_count = 0\n",
    "total_num = 0\n",
    "for i in range(total_batch):\n",
    "    num_valid = batch_size if (i + 1) * batch_size <= X_test.shape[0]\\\n",
    "                           else X_test.shape[0] - i * batch_size\n",
    "    data_npy = np.take(X_test,\n",
    "                       indices=np.arange(i * batch_size, (i + 1) * batch_size),\n",
    "                       axis=0,\n",
    "                       mode=\"clip\")\n",
    "    label_npy = np.take(y_test,\n",
    "                        indices=np.arange(i * batch_size, (i + 1) * batch_size),\n",
    "                        axis=0,\n",
    "                        mode=\"clip\")\n",
    "    test_net.forward(data_batch=mx.io.DataBatch(data=[nd.array(data_npy)],\n",
    "                                                label=None),\n",
    "                     is_train=False)\n",
    "    logits_nd = test_net.get_outputs()[0]\n",
    "    pred_cls = nd.argmax(logits_nd, axis=-1).asnumpy()\n",
    "    correct_count += (pred_cls[:num_valid] == label_npy[:num_valid]).sum()\n",
    "acc = correct_count / float(X_test.shape[0])\n",
    "print('Accuracy:', acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> 6. Get one and predict </h3>\n",
    "\n",
    "We can predict the label of a single sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label:  2\nPrediction:  2\n"
     ]
    }
   ],
   "source": [
    "test_net.reshape(data_shapes=[mx.io.DataDesc(name='data', shape=(1, 1, 28, 28), layout='NCHW')],\n",
    "                 label_shapes=None)\n",
    "r = np.random.randint(0, X_test.shape[0])\n",
    "test_net.forward(data_batch=mx.io.DataBatch(data=[nd.array(X_test[r:r+1])],\n",
    "                                            label=None))\n",
    "logits_nd = test_net.get_outputs()[0]\n",
    "print(\"Label: \", int(y_test[r]))\n",
    "print(\"Prediction: \", int(nd.argmax(logits_nd, axis=1).asnumpy()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}