example/notebooks/simple_bind.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MXNet Symbol.simple_bind example\n",
    "\n",
    "In this example, we will show how to use ```simple_bind``` API. \n",
    "\n",
    "Note it is a low level API.  By using such a low level API, we are able to interact with more details about MXNet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "import numpy as np\n",
    "import logging\n",
    "import pprint\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a very simple 1 hidden layer BatchNorm fully connected MNIST network to demo how to use low level API.\n",
    "The network looks like:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.36.0 (20140111.2315)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"144pt\" height=\"616pt\"\n",
       " viewBox=\"0.00 0.00 144.00 616.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 612)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-612 140,-612 140,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"68\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-25.3\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fc1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fc1</title>\n",
       "<ellipse fill=\"#fb8072\" stroke=\"black\" cx=\"68\" cy=\"-139\" rx=\"68.2532\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-142.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-127.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc1&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fc1&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-99.8131C68,-86.1516 68,-71.0092 68,-58.3283\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-109.906 63.5001,-99.9062 68,-104.906 68.0001,-99.9062 68.0001,-99.9062 68.0001,-99.9062 68,-104.906 72.5001,-99.9062 68,-109.906 68,-109.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-80.3\" font-family=\"Times,serif\" font-size=\"14.00\">784</text>\n",
       "</g>\n",
       "<!-- bn1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>bn1</title>\n",
       "<ellipse fill=\"#bebada\" stroke=\"black\" cx=\"68\" cy=\"-249\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-245.3\" font-family=\"Times,serif\" font-size=\"14.00\">BatchNorm</text>\n",
       "</g>\n",
       "<!-- bn1&#45;&gt;fc1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>bn1&#45;&gt;fc1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-209.813C68,-196.152 68,-181.009 68,-168.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-219.906 63.5001,-209.906 68,-214.906 68.0001,-209.906 68.0001,-209.906 68.0001,-209.906 68,-214.906 72.5001,-209.906 68,-219.906 68,-219.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-190.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- act1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>act1</title>\n",
       "<ellipse fill=\"#ffffb3\" stroke=\"black\" cx=\"68\" cy=\"-359\" rx=\"48.4635\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-362.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-347.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- act1&#45;&gt;bn1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>act1&#45;&gt;bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-319.813C68,-306.152 68,-291.009 68,-278.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-329.906 63.5001,-319.906 68,-324.906 68.0001,-319.906 68.0001,-319.906 68.0001,-319.906 68,-324.906 72.5001,-319.906 68,-329.906 68,-329.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-300.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>fc2</title>\n",
       "<ellipse fill=\"#fb8072\" stroke=\"black\" cx=\"68\" cy=\"-469\" rx=\"68.2532\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-472.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-457.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- fc2&#45;&gt;act1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>fc2&#45;&gt;act1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-429.813C68,-416.152 68,-401.009 68,-388.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-439.906 63.5001,-429.906 68,-434.906 68.0001,-429.906 68.0001,-429.906 68.0001,-429.906 68,-434.906 72.5001,-429.906 68,-439.906 68,-439.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-410.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- softmax -->\n",
       "<g id=\"node6\" class=\"node\"><title>softmax</title>\n",
       "<ellipse fill=\"#fccde5\" stroke=\"black\" cx=\"68\" cy=\"-579\" rx=\"55.0152\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-575.3\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;fc2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>softmax&#45;&gt;fc2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-539.813C68,-526.152 68,-511.009 68,-498.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-549.906 63.5001,-539.906 68,-544.906 68.0001,-539.906 68.0001,-539.906 68.0001,-539.906 68,-544.906 72.5001,-539.906 68,-549.906 68,-549.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"75\" y=\"-520.3\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fe7b800ad10>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can use mx.sym in short of mx.symbol\n",
    "data = mx.sym.Variable(\"data\")\n",
    "fc1 = mx.sym.FullyConnected(data=data, num_hidden=128, name=\"fc1\")\n",
    "bn1 = mx.sym.BatchNorm(data=fc1, name=\"bn1\")\n",
    "act1 = mx.sym.Activation(data=bn1, name=\"act1\", act_type=\"tanh\")\n",
    "fc2 = mx.sym.FullyConnected(data=act1, name=\"fc2\", num_hidden=10)\n",
    "softmax = mx.sym.Softmax(data=fc2, name=\"softmax\")\n",
    "# visualize the network\n",
    "batch_size = 100\n",
    "data_shape = (batch_size, 784)\n",
    "mx.viz.plot_network(softmax, shape={\"data\":data_shape}, node_attrs={\"shape\":'oval',\"fixedsize\":'false'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use ```simple_bind``` api to generate executor from symbol."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# context different to ```mx.model```, \n",
    "# In mx.model, we wrapped parameter server, but for a single executor, the context is only able to be ONE device\n",
    "# run on cpu\n",
    "ctx = mx.cpu()\n",
    "# run on gpu\n",
    "# ctx = mx.gpu()\n",
    "# run on third gpu\n",
    "# ctx = mx.gpu(2)\n",
    "executor = softmax.simple_bind(ctx=ctx, data=data_shape, grad_req='write')\n",
    "# The default ctx is CPU, data's shape is required and ```simple_bind``` will try to infer all other required \n",
    "# For MLP, the ```grad_req``` is write to, and for RNN it is different"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After generating executor, get lists of inputs (args), outputs, etc\n",
    "The order of these arrays is in same sequence as symbol arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "args: {'bn1_beta': <mxnet.ndarray.NDArray object at 0x7fe7b7d146d0>,\n",
      " 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7fe7b7d147d0>,\n",
      " 'data': <mxnet.ndarray.NDArray object at 0x7fe804ad70d0>,\n",
      " 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7fe7b7d14590>,\n",
      " 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7fe7b7d14750>,\n",
      " 'fc2_bias': <mxnet.ndarray.NDArray object at 0x7fe7b7d14650>,\n",
      " 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7fe7b7d14690>,\n",
      " 'softmax_label': <mxnet.ndarray.NDArray object at 0x7fe7b7d14610>}\n",
      "--------------------\n",
      "grads: {'bn1_beta': <mxnet.ndarray.NDArray object at 0x7fe7b7d14910>,\n",
      " 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7fe7b7d148d0>,\n",
      " 'data': <mxnet.ndarray.NDArray object at 0x7fe7b7d14810>,\n",
      " 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7fe7b7d14890>,\n",
      " 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7fe7b7d14850>,\n",
      " 'fc2_bias': <mxnet.ndarray.NDArray object at 0x7fe7b7d14990>,\n",
      " 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7fe7b7d14950>,\n",
      " 'softmax_label': <mxnet.ndarray.NDArray object at 0x7fe7b7d149d0>}\n",
      "--------------------\n",
      "aux_states: {'bn1_moving_mean': <mxnet.ndarray.NDArray object at 0x7fe7b7d14a10>,\n",
      " 'bn1_moving_var': <mxnet.ndarray.NDArray object at 0x7fe7b7d14a50>}\n",
      "--------------------\n",
      "outputs: {'softmax_output': <mxnet.ndarray.NDArray object at 0x7fe7b7d14c10>}\n"
     ]
    }
   ],
   "source": [
    "args = executor.arg_dict\n",
    "# Equivalently you could do this:\n",
    "#args = dict(zip(softmax.list_arguments(), executor.arg_arrays))\n",
    "grads = executor.grad_dict\n",
    "aux_states = executor.aux_dict\n",
    "\n",
    "# For outputs we need to assemble the dict by hand:\n",
    "outputs = dict(zip(softmax.list_outputs(), executor.outputs))\n",
    "\n",
    "# we can print the args we have\n",
    "print(\"args: %s\" % pprint.pformat(args))\n",
    "print(\"-\" * 20)\n",
    "print(\"grads: %s\" % pprint.pformat(grads))\n",
    "print(\"-\" * 20)\n",
    "print(\"aux_states: %s\" % pprint.pformat(aux_states))\n",
    "print(\"-\" * 20)\n",
    "print(\"outputs: %s\" % pprint.pformat(outputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is intilize the weights. We can set weight directly by using ```mx.random``` or numpy ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "args['fc1_weight'][:] = mx.random.uniform(-0.07, 0.07, args['fc1_weight'].shape)\n",
    "args['fc2_weight'][:] = np.random.uniform(-0.07, 0.07, args['fc2_weight'].shape)  # equivalent\n",
    "args['bn1_beta'][:] = 1.0\n",
    "args['bn1_gamma'][:] = 1.0\n",
    "args['fc1_bias'][:] = 0\n",
    "args['fc2_bias'][:] = 0\n",
    "# Don't initialize data or softmax_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can customize our update rule. Here for demo purpose, we make a simple update rule to show mxnet feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SGD(key, weight, grad, lr=0.1, grad_norm=batch_size):\n",
    "    # key is key for weight, we can customize update rule\n",
    "    # weight is weight array\n",
    "    # grad is grad array\n",
    "    # lr is learning rate\n",
    "    # grad_norm is scalar to norm gradient, usually it is batch_size\n",
    "    norm = 1.0 / grad_norm\n",
    "    # here we can bias' learning rate 2 times larger than weight\n",
    "    if \"weight\" in key or \"gamma\" in key:\n",
    "        weight[:] -= lr * (grad * norm)\n",
    "    elif \"bias\" in key or \"beta\" in key:\n",
    "        weight[:] -= 2.0 * lr * (grad * norm)\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we will make a data iterator. We can either use built-in iterator to load from binary file or build a numpy iterator.\n",
    "\n",
    "For special case, you are free to write your own iterator in python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/numpy/core/fromnumeric.py:2652: VisibleDeprecationWarning: `rank` is deprecated; use the `ndim` attribute or function instead. To find the rank of a matrix see `numpy.linalg.matrix_rank`.\n",
      "  VisibleDeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# We use utils function in sklearn to get MNIST dataset in pickle\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.utils import shuffle\n",
    "mnist = fetch_mldata('MNIST original', data_home=\"./data\")\n",
    "# shuffle data\n",
    "X, y = shuffle(mnist.data, mnist.target)\n",
    "# split dataset\n",
    "train_data = X[:50000, :].astype('float32')\n",
    "train_label = y[:50000]\n",
    "val_data = X[50000: 60000, :].astype('float32')\n",
    "val_label = y[50000:60000]\n",
    "# Normalize data\n",
    "train_data[:] /= 256.0\n",
    "val_data[:] /= 256.0\n",
    "# Build iterator\n",
    "train_iter = mx.io.NDArrayIter(data=train_data, label=train_label, batch_size=batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(data=val_data, label=val_label, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, we need to define a helper function to calculate accuracy of current training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Accuracy(label, pred_prob):\n",
    "    pred = np.argmax(pred_prob, axis=1)\n",
    "    return np.sum(label == pred) * 1.0 / label.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can train the network by using executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Finish training iteration 0\n",
      "INFO:root:Train Acc: 0.8894\n",
      "INFO:root:Val Acc: 0.9319\n",
      "INFO:root:Finish training iteration 1\n",
      "INFO:root:Train Acc: 0.9372\n",
      "INFO:root:Val Acc: 0.9472\n",
      "INFO:root:Finish training iteration 2\n",
      "INFO:root:Train Acc: 0.9508\n",
      "INFO:root:Val Acc: 0.9541\n"
     ]
    }
   ],
   "source": [
    "num_round = 3\n",
    "keys = softmax.list_arguments()\n",
    "# we use extra ndarray to save output of net\n",
    "pred_prob = mx.nd.zeros(executor.outputs[0].shape)\n",
    "for i in range(num_round):\n",
    "    train_iter.reset()\n",
    "    val_iter.reset()\n",
    "    train_acc = 0.\n",
    "    val_acc = 0.\n",
    "    nbatch = 0.\n",
    "    # train\n",
    "    for dbatch in train_iter:\n",
    "        data = dbatch.data[0]\n",
    "        label = dbatch.label[0]\n",
    "        # copy data into args\n",
    "        args[\"data\"][:] = data # or we can ```data.copyto(args[\"data\"])```\n",
    "        args[\"softmax_label\"][:] = label\n",
    "        executor.forward(is_train=True)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        executor.backward()\n",
    "        for key in keys:\n",
    "            SGD(key, args[key], grads[key])\n",
    "        train_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    logging.info(\"Finish training iteration %d\" % i)\n",
    "    train_acc /= nbatch\n",
    "    nbatch = 0.\n",
    "    # eval\n",
    "    for dbatch in val_iter:\n",
    "        data = dbatch.data[0]\n",
    "        label = dbatch.label[0]\n",
    "        args[\"data\"][:] = data\n",
    "        executor.forward(is_train=False)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        val_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    val_acc /= nbatch\n",
    "    logging.info(\"Train Acc: %.4f\" % train_acc)\n",
    "    logging.info(\"Val Acc: %.4f\" % val_acc)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "This is simple example gives a demo on how to directly use symbolic API to build a neural net from scratch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}