nbs/lecture_notes/01_building_micrograd.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mircrograd from scratch\n",
    " - These notebooks/docs are created when going through the lessons of zero to hero taught by andrej karpathy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from rich import print\n",
    "from rich import pretty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretty.install()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single Variable: `x`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x): return 3*x**2 - 4*x + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m20.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f(3.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">array</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-5</span>.  , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-4.75</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-4.5</span> , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-4.25</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-4</span>.  , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.75</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.5</span> , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.25</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3</span>.  ,\n",
       "       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.75</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.5</span> , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.25</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2</span>.  , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.75</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.5</span> , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.25</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1</span>.  , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.75</span>,\n",
       "       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5</span> , <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>.  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5</span> ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.75</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>.  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.5</span> ,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.75</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>.  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.5</span> ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.75</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>.  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.5</span> ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.75</span>,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>.  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.25</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.5</span> ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.75</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35marray\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-5\u001b[0m.  , \u001b[1;36m-4.75\u001b[0m, \u001b[1;36m-4.5\u001b[0m , \u001b[1;36m-4.25\u001b[0m, \u001b[1;36m-4\u001b[0m.  , \u001b[1;36m-3.75\u001b[0m, \u001b[1;36m-3.5\u001b[0m , \u001b[1;36m-3.25\u001b[0m, \u001b[1;36m-3\u001b[0m.  ,\n",
       "       \u001b[1;36m-2.75\u001b[0m, \u001b[1;36m-2.5\u001b[0m , \u001b[1;36m-2.25\u001b[0m, \u001b[1;36m-2\u001b[0m.  , \u001b[1;36m-1.75\u001b[0m, \u001b[1;36m-1.5\u001b[0m , \u001b[1;36m-1.25\u001b[0m, \u001b[1;36m-1\u001b[0m.  , \u001b[1;36m-0.75\u001b[0m,\n",
       "       \u001b[1;36m-0.5\u001b[0m , \u001b[1;36m-0.25\u001b[0m,  \u001b[1;36m0\u001b[0m.  ,  \u001b[1;36m0.25\u001b[0m,  \u001b[1;36m0.5\u001b[0m ,  \u001b[1;36m0.75\u001b[0m,  \u001b[1;36m1\u001b[0m.  ,  \u001b[1;36m1.25\u001b[0m,  \u001b[1;36m1.5\u001b[0m ,\n",
       "        \u001b[1;36m1.75\u001b[0m,  \u001b[1;36m2\u001b[0m.  ,  \u001b[1;36m2.25\u001b[0m,  \u001b[1;36m2.5\u001b[0m ,  \u001b[1;36m2.75\u001b[0m,  \u001b[1;36m3\u001b[0m.  ,  \u001b[1;36m3.25\u001b[0m,  \u001b[1;36m3.5\u001b[0m ,  \u001b[1;36m3.75\u001b[0m,\n",
       "        \u001b[1;36m4\u001b[0m.  ,  \u001b[1;36m4.25\u001b[0m,  \u001b[1;36m4.5\u001b[0m ,  \u001b[1;36m4.75\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = np.arange(-5, 5, 0.25)\n",
    "xs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">array</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">91.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">83.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">76.1875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">69</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62.1875</span>,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">55.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">49.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">44</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">38.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">29.1875</span>,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">25</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">21.1875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>.    ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9.6875</span>,\n",
       "         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7.75</span>  ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.1875</span>,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>.    ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.1875</span>,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.75</span>  ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.6875</span>,\n",
       "         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>.    ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.6875</span>,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5.75</span>  ,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7.1875</span>,   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11.1875</span>,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23.6875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32.1875</span>,\n",
       "        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">37</span>.    ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">42.1875</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">47.75</span>  ,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">53.6875</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35marray\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m100\u001b[0m.    ,  \u001b[1;36m91.6875\u001b[0m,  \u001b[1;36m83.75\u001b[0m  ,  \u001b[1;36m76.1875\u001b[0m,  \u001b[1;36m69\u001b[0m.    ,  \u001b[1;36m62.1875\u001b[0m,\n",
       "        \u001b[1;36m55.75\u001b[0m  ,  \u001b[1;36m49.6875\u001b[0m,  \u001b[1;36m44\u001b[0m.    ,  \u001b[1;36m38.6875\u001b[0m,  \u001b[1;36m33.75\u001b[0m  ,  \u001b[1;36m29.1875\u001b[0m,\n",
       "        \u001b[1;36m25\u001b[0m.    ,  \u001b[1;36m21.1875\u001b[0m,  \u001b[1;36m17.75\u001b[0m  ,  \u001b[1;36m14.6875\u001b[0m,  \u001b[1;36m12\u001b[0m.    ,   \u001b[1;36m9.6875\u001b[0m,\n",
       "         \u001b[1;36m7.75\u001b[0m  ,   \u001b[1;36m6.1875\u001b[0m,   \u001b[1;36m5\u001b[0m.    ,   \u001b[1;36m4.1875\u001b[0m,   \u001b[1;36m3.75\u001b[0m  ,   \u001b[1;36m3.6875\u001b[0m,\n",
       "         \u001b[1;36m4\u001b[0m.    ,   \u001b[1;36m4.6875\u001b[0m,   \u001b[1;36m5.75\u001b[0m  ,   \u001b[1;36m7.1875\u001b[0m,   \u001b[1;36m9\u001b[0m.    ,  \u001b[1;36m11.1875\u001b[0m,\n",
       "        \u001b[1;36m13.75\u001b[0m  ,  \u001b[1;36m16.6875\u001b[0m,  \u001b[1;36m20\u001b[0m.    ,  \u001b[1;36m23.6875\u001b[0m,  \u001b[1;36m27.75\u001b[0m  ,  \u001b[1;36m32.1875\u001b[0m,\n",
       "        \u001b[1;36m37\u001b[0m.    ,  \u001b[1;36m42.1875\u001b[0m,  \u001b[1;36m47.75\u001b[0m  ,  \u001b[1;36m53.6875\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ys = f(xs)\n",
    "ys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">matplotlib.lines.Line2D</span><span style=\"color: #000000; text-decoration-color: #000000\"> object at </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0x7fb25cd152d0</span><span style=\"font-weight: bold\">&gt;]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\u001b[1m<\u001b[0m\u001b[1;95mmatplotlib.lines.Line2D\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7fb25cd152d0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABC4ElEQVR4nO3deVxU5eIG8OfMDAzbMAjCjAgoIq64YuJWYiValluLpS1Wt/RqFllZZovZvZJW5k2vlv3KLHO5dd1aNC1LUzRxQQVXFARBRARnWGdg5vz+QOeGmusM7yzP9/M5n5tnzuDj3D7O03ve876SLMsyiIiIiJyIQnQAIiIioouxoBAREZHTYUEhIiIip8OCQkRERE6HBYWIiIicDgsKEREROR0WFCIiInI6LChERETkdFSiA9wIq9WKgoICaDQaSJIkOg4RERFdA1mWUVZWhvDwcCgUVx4jccmCUlBQgMjISNExiIiI6Abk5eUhIiLiite4ZEHRaDQA6v6AgYGBgtMQERHRtTAajYiMjLR9j1+JSxaUC7d1AgMDWVCIiIhczLVMz+AkWSIiInI6LChERETkdFhQiIiIyOmwoBAREZHTYUEhIiIip8OCQkRERE6HBYWIiIicDgsKEREROR0WFCIiInI6111QNm/ejHvvvRfh4eGQJAmrVq2q97osy5g6dSrCw8Ph6+uLxMREZGZm1rvGZDJhwoQJaNy4Mfz9/TF48GCcPHnypv4gRERE5D6uu6BUVFSgU6dOmDt37mVfnzlzJmbNmoW5c+ciLS0Ner0e/fv3R1lZme2a5ORkrFy5EsuWLcOWLVtQXl6Oe+65BxaL5cb/JEREROQ2JFmW5Rt+syRh5cqVGDp0KIC60ZPw8HAkJyfjlVdeAVA3WqLT6TBjxgyMGTMGBoMBoaGh+OqrrzBixAgA/9ud+Mcff8SAAQOu+vsajUZotVoYDAbuxUNEROQiruf7265zULKzs1FYWIikpCTbObVajb59+yI1NRUAsGvXLtTU1NS7Jjw8HHFxcbZrLmYymWA0GusdjnCmzISPfjmKmesOOeTnExER0bWxa0EpLCwEAOh0unrndTqd7bXCwkJ4e3ujUaNGf3nNxVJSUqDVam1HZGSkPWPb5JZUYtaGI/h8azYMVTUO+T2IiIjo6hzyFM/F2yjLsnzVrZWvdM3kyZNhMBhsR15ent2y/lnXqCC01mlQXWPF6vR8h/weREREdHV2LSh6vR4ALhkJKSoqso2q6PV6mM1mlJaW/uU1F1Or1QgMDKx3OIIkSXi4e93ozJI/cnET03OIiIjoJti1oERHR0Ov12PDhg22c2azGZs2bUKvXr0AAPHx8fDy8qp3zalTp5CRkWG7RqRhXSKgVilwqLAM6XnnRMchIiLySKrrfUN5eTmysrJsv87OzkZ6ejqCg4MRFRWF5ORkTJ8+HbGxsYiNjcX06dPh5+eHkSNHAgC0Wi2eeuopvPjiiwgJCUFwcDBeeukldOjQAXfeeaf9/mQ3SOvnhUEdm2DF7nws3ZGLLlGNrv4mIiIisqvrLig7d+5Ev379bL+eOHEiAODxxx/HF198gUmTJqGqqgrjxo1DaWkpEhISsH79emg0Gtt7PvzwQ6hUKjz44IOoqqrCHXfcgS+++AJKpdIOf6SbN7J7FFbszsd3e0/h9XvaIdDHS3QkIiIij3JT66CI4uh1UGRZxoDZm3HkdDneGRqHR3s0s/vvQURE5GmErYPiLuomy0YB4GRZIiIiEVhQ/sKwLk2hVilw8JQRe08aRMchIiLyKCwofyHIzxuDOjQBACz9I1dwGiIiIs/CgnIFDyfU3eZZs7cAZdVcWZaIiKihsKBcQbdmjdAyLABVNRasTi8QHYeIiMhjsKBcASfLEhERicGCchXDuzSFt0qBA6eM2J/PybJEREQNgQXlKhr5e+PuuLo9hpbu4GRZIiKihsCCcg0u3OZZnV6AclOt4DRERETujwXlGnSPDkaLUH9Umi1Yw8myREREDseCcg0kScLI86MovM1DRETkeCwo12h41wh4KxXYn2/Afq4sS0RE5FAsKNco2N8bAy9Mlk3jKAoREZEjsaBcB9tk2T35qOBkWSIiIodhQbkOPVoEI7qxPyrMFqzZy8myREREjsKCch3qVpaNBMDJskRERI7EgnKd7usaAS+lhH0nDcjgyrJEREQOwYJynUIC1BjQnivLEhERORILyg0Y+aeVZTlZloiIyP5YUG5AjxYhaB7ih3JTLb7fx8myRERE9saCcgMUCsn2yPGSHXmC0xAREbkfFpQbdF983WTZvXnnkFnAybJERET2xIJygxoHqJF0frLs139wsiwREZE9saDchFEJdbd5Vu3Jh7G6RnAaIiIi98GCchN6tghBbFgAKs0WrNh1UnQcIiIit8GCchMkScKjPZsBAL7afgKyLAtORERE5B5YUG7SsC5N4e+txLEzFUg9dlZ0HCIiIrfAgnKTND5eGN41AgDw5bYcsWGIiIjcBAuKHVy4zbPhwGkUnKsSnIaIiMj1saDYQSudBj1aBMMqA0v4yDEREdFNY0Gxk8d6NgcALEvLhanWIjYMERGRi2NBsZP+7XTQBapRXG7GuoxC0XGIiIhcGguKnXgpFbb9eb7adkJwGiIiItfGgmJHI7tHQaWQsPNEKQ4UGEXHISIiclksKHYUFuiDAXF1+/N8tT1HbBgiIiIXxoJiZ4/1qHvkeNWeAhiquD8PERHRjWBBsbPu0cFordOgqsaCb7k/DxER0Q1hQbGzP+/Ps3j7CVit3J+HiIjoerGgOMCwLk2hUauQXVyBLVnFouMQERG5HBYUB/BXq3Bf/IX9efjIMRER0fViQXGQR85Plt146DROllYKTkNERORaWFAcpGVYAHq3DOH+PERERDeABcWBHj0/irI8LY/78xAREV0HFhQHurOtDk20PjhbYcaP+0+JjkNEROQyWFAcSKVUYOT5/Xk4WZaIiOjasaA42EPdo+CllLAn9xwy8g2i4xAREbkEFhQHC9WocVdcEwDAl9tyxIYhIiJyESwoDeCx8yvLrk4vwLlKs+A0REREzo8FpQHEN2uEtk0CYaq14pud3J+HiIjoalhQGoAkSbZRlMV/cH8eIiKiq2FBaSBDOocj0EeFE2cr8evhItFxiIiInBoLSgPx81bh4fOPHH++NVtwGiIiIufGgtKAHuvVHEqFhK1ZZ3Go0Cg6DhERkdNiQWlATYN8MbC9HgCwcEuO2DBEREROjAWlgT3ZpzkAYGV6PorLTWLDEBEROSkWlAbWNaoROkUGwVxr5S7HREREf4EFpYFJkoQnezcHAHy1/QR3OSYiIroMFhQB7u7QBLpANc6UmfDDPu5yTEREdDEWFAG8lAo81rM5AOCzLdmQZS7cRkRE9Gd2Lyi1tbV4/fXXER0dDV9fX7Ro0QLTpk2D1Wq1XSPLMqZOnYrw8HD4+voiMTERmZmZ9o7i1EZ2j4KPlwKZBUbsyC4RHYeIiMip2L2gzJgxAx9//DHmzp2LgwcPYubMmXjvvfcwZ84c2zUzZ87ErFmzMHfuXKSlpUGv16N///4oKyuzdxyn1cjfG8O7RgDgwm1EREQXs3tB2bZtG4YMGYJBgwahefPmuP/++5GUlISdO3cCqBs9mT17NqZMmYLhw4cjLi4OixYtQmVlJZYsWWLvOE7tiV7NAQDrD5xG7tlKsWGIiIiciN0LSp8+ffDLL7/gyJEjAIC9e/diy5YtuPvuuwEA2dnZKCwsRFJSku09arUaffv2RWpq6mV/pslkgtForHe4g1idBre1CoUsA4u25YiOQ0RE5DTsXlBeeeUVPPzww2jTpg28vLzQpUsXJCcn4+GHHwYAFBYWAgB0Ol299+l0OttrF0tJSYFWq7UdkZGR9o4tzIVHjpen5aGsukZsGCIiIidh94KyfPlyLF68GEuWLMHu3buxaNEivP/++1i0aFG96yRJqvdrWZYvOXfB5MmTYTAYbEdeXp69YwtzW2woYkL9UW6qxbe7ToqOQ0RE5BTsXlBefvllvPrqq3jooYfQoUMHPProo3jhhReQkpICANDr6/aiuXi0pKio6JJRlQvUajUCAwPrHe5CoZDwZJ9oAMDCrTmwWPnIMRERkd0LSmVlJRSK+j9WqVTaHjOOjo6GXq/Hhg0bbK+bzWZs2rQJvXr1sncclzC8SwS0vl7ILanELwdPi45DREQknN0Lyr333ot//vOf+OGHH5CTk4OVK1di1qxZGDZsGIC6WzvJycmYPn06Vq5ciYyMDIwePRp+fn4YOXKkveO4BF9vJUYmRAHgI8dEREQAoLL3D5wzZw7eeOMNjBs3DkVFRQgPD8eYMWPw5ptv2q6ZNGkSqqqqMG7cOJSWliIhIQHr16+HRqOxdxyX8VjPZvh083FsP16CzAID2odrRUciIiISRpJdcJ11o9EIrVYLg8HgVvNRnlu6B2v2FuD++Ai8/0An0XGIiIjs6nq+v7kXjxO5MFl2TXoBisqqBachIiIShwXFiXSODELXqCCYLVZ8vT1XdBwiIiJhWFCczIVRlK//OIHqGovgNERERGKwoDiZge31CNf6oLjcjO/2FoiOQ0REJAQLipNRKRV4/Pwmgp9vzYELzmEmIiK6aSwoTuihW6Lg66XEwVNGbD9eIjoOERFRg2NBcUJaPy/cHx8BAPj09+OC0xARETU8FhQn9WSfaEgSsPFQEY6cLhMdh4iIqEGxoDip6Mb+GNi+bmPFBZs5ikJERJ6FBcWJPXNbCwDA6vR8nDJUCU5DRETUcFhQnFiXqEZIiA5GjUXGwq05ouMQERE1GBYUJze2bwwAYMkfuTBU1QhOQ0RE1DBYUJxcYutQtNZpUG6qxZI/uPw9ERF5BhYUJydJkm0uyudbs2Gq5fL3RETk/lhQXMDgzuEI1/rgTJkJq/bki45DRETkcCwoLsBLqbBtIvjJ5uOwWrn8PRERuTcWFBfxUPcoBPqocPxMBX4+eFp0HCIiIodiQXERAWoVHu3ZDEDdKAoREZE7Y0FxIY/3ag5vlQK7TpQiLYebCBIRkftiQXEhYRof3Ne1bhPBTzYdE5yGiIjIcVhQXMzTt9ZtIvjzwSIc5SaCRETkplhQXEyL0AAMaMdNBImIyL2xoLigMX3rFm5blZ6PQkO14DRERET2x4LigrpENUJ32yaC2aLjEBER2R0Liosae34U5es/cmGs5iaCRETkXlhQXFRiqzC00gVwE0EiInJLLCguSqGQ8MxtMQCAz7dwE0EiInIvLCgubHCncOgDfVBUZsLqPQWi4xAREdkNC4oL81Yp8JRtE8Fj3ESQiIjcBguKi3uoeyQ0PiocO1OBXw4ViY5DRERkFywoLk7j44VHepzfRJDL3xMRkZtgQXEDT/RqDm+lAjtPlGJHNjcRJCIi18eC4gbCAn1wf7e6TQTnbDwqOA0REdHNY0FxE3/vGwOlQsLvR4uRnndOdBwiIqKbwoLiJiKD/TCsS1MAwFyOohARkYtjQXEj4xJjoJCAnw8WIbPAIDoOERHRDWNBcSMtQgNwT8dwAMC/f80SnIaIiOjGsaC4mfH9WgIA1mYU4ujpMsFpiIiIbgwLiptprddgYHs9ZJmjKERE5LpYUNzQs7fXjaKs2VuA7OIKwWmIiIiuHwuKG4prqsXtbcJglYH5v3EUhYiIXA8Lipu6MIqyYnc+8koqBachIiK6PiwobqprVCP0adkYtVYZn2zmHj1ERORaWFDc2IVRlP+knUShoVpwGiIiomvHguLGerQIQffmwTBbrFiw+bjoOERERNeMBcXNXRhFWbLjBIrLTYLTEBERXRsWFDd3a2xjdIoMQnWNFZ/+zlEUIiJyDSwobk6SJEw4v7rs4m0nUFphFpyIiIjo6lhQPMAdbcPQtkkgKswWLNyaLToOERHRVbGgeABJkjDh/FyUhak5MFbXCE5ERER0ZSwoHmJgez1ahgWgrLoWX6bmiI5DRER0RSwoHkKhkPDs+bkon23JRoWpVnAiIiKiv8aC4kHu6dgEzUP8UFpZg6//OCE6DhER0V9iQfEgKqUC4xLrRlEWbM5GdY1FcCIiIqLLY0HxMMO6NkXTIF8Ul5uwbEeu6DhERESXxYLiYbyUCoxNjAEAzN90jKMoRETklFhQPNCD3SLQNMgXp40mLN7OuShEROR8WFA8kFqlxHN31M1Fmf/bMT7RQ0RETocFxUMN7xqBZiF+OFthxhdcF4WIiJyMQwpKfn4+HnnkEYSEhMDPzw+dO3fGrl27bK/LsoypU6ciPDwcvr6+SExMRGZmpiOi0F/wUiqQfGcsAGDB5uNcXZaIiJyK3QtKaWkpevfuDS8vL6xduxYHDhzABx98gKCgINs1M2fOxKxZszB37lykpaVBr9ejf//+KCsrs3ccuoLBnZqiZVgADFU1+Ox37tFDRETOQ5JlWbbnD3z11VexdetW/P7775d9XZZlhIeHIzk5Ga+88goAwGQyQafTYcaMGRgzZsxVfw+j0QitVguDwYDAwEB7xvc4P+4/hXFf70aAWoXfJ/VDI39v0ZGIiMhNXc/3t91HUNasWYNu3brhgQceQFhYGLp06YJPP/3U9np2djYKCwuRlJRkO6dWq9G3b1+kpqZe9meaTCYYjcZ6B9nHwPZ6tGsSiHJTLT7ZfFx0HCIiIgAOKCjHjx/H/PnzERsbi59++gljx47Fc889hy+//BIAUFhYCADQ6XT13qfT6WyvXSwlJQVardZ2REZG2ju2x1IoJEzs3woAsCg1B2fKTIITEREROaCgWK1WdO3aFdOnT0eXLl0wZswYPP3005g/f3696yRJqvdrWZYvOXfB5MmTYTAYbEdeXp69Y3u0O9qGoVNkEKpqLJj/2zHRcYiIiOxfUJo0aYJ27drVO9e2bVvk5tYtq67X6wHgktGSoqKiS0ZVLlCr1QgMDKx3kP1IkoSXkupGURb/cQKnDFWCExERkaeze0Hp3bs3Dh8+XO/ckSNH0KxZMwBAdHQ09Ho9NmzYYHvdbDZj06ZN6NWrl73j0DXq07IxukcHw1xrxdyNWaLjEBGRh7N7QXnhhRewfft2TJ8+HVlZWViyZAkWLFiA8ePHA6j7r/Xk5GRMnz4dK1euREZGBkaPHg0/Pz+MHDnS3nHoGkmShBfPz0VZnpaHvJJKwYmIiMiT2b2g3HLLLVi5ciWWLl2KuLg4vPPOO5g9ezZGjRplu2bSpElITk7GuHHj0K1bN+Tn52P9+vXQaDT2jkPXIaFFCG6NbYxaq4x//XJUdBwiIvJgdl8HpSFwHRTH2ZNbimHzUqGQgJ8n9kWL0ADRkYiIyE0IXQeFXFuXqEa4s20YrDIw+2eOohARkRgsKHSJF87PRfluXwEOFXJRPCIiangsKHSJ9uFaDOrQBLIMfLjhiOg4RETkgVhQ6LKS74yFJAE/ZZ7G/pMG0XGIiMjDsKDQZcXqNBjauSkAYNaGw1e5moiIyL5YUOgvPX9HLJQKCb8ePoNdJ0pExyEiIg/CgkJ/qXljfzwQHwEA+GA956IQEVHDYUGhK5pwRyy8lQqkHjuLrVnFouMQEZGHYEGhK2oa5IuRCVEAgJS1B2G1uty6fkRE5IJYUOiqJtzeEgFqFTLyjfhuX4HoOERE5AFYUOiqQgLU+HtiDADgvZ8Ow1RrEZyIiIjcHQsKXZMne0dDF6jGydIqfLXthOg4RETk5lhQ6Jr4eivxYv/WAIA5G7NgqKwRnIiIiNwZCwpds/viI9BKFwBDVQ3mbcoSHYeIiBzk6OkyyLLYhyJYUOiaKRUSJt/VFgCwcGsO8s9VCU5ERET2lldSiUFztmDEJ9thrBY3Ws6CQtclsXUoerYIgbnWig/Wcwl8IiJ3895Ph2GutUKpkKBRq4TlYEGh6yJJEibf3QYAsHJPPjILuJEgEZG7SM87hzV7CyBJwJRBbSFJkrAsLCh03TpGBOHeTuGQZeDdtYdExyEiIjuQZRn/+P4AAGB4lwjENdUKzcOCQjfk5aTW8FJK+P1oMTYfOSM6DhER3aR1GYXYeaIUPl4KvDygteg4LCh0Y6JC/PBoj+YAgJS1h7gEPhGRCzPXWvHuuroR8WdubQG91kdwIhYUugkTbm8JjY8KB08ZsSo9X3QcIiK6QV9uy8GJs5UI1agxpm+M6DgAWFDoJjTy98a4xJYAgPd/OozqGi6BT0Tkas5VmjFnY93aVi/2bwV/gU/u/BkLCt2UJ3o3RxOtDwoM1ViUmiM6DhERXaePfsmCoaoGbfQaPNAtUnQcGxYUuik+Xkq8mFQ3merfv2bhXKVZcCIiIrpWOcUV+Gp7DgDgtbvbQqkQ91jxxVhQ6KYN69IUbfQaGKtr8e9fuQQ+EZGrmLHuEGosMm5rFYrbWoWKjlMPCwrdNKVCwuS765bAX5R6AnkllYITERHR1aTllGBtRiEUEjDl/N/hzoQFhezittjG6NOyMcwWLoFPROTsrFYZ//jhIABgxC2RaK3XCE50KRYUsgtJkvDqXXVL4K9KL0BGPpfAJyJyVt/tK8DevHPw91bihf6tRMe5LBYUspu4ploM69IUAPDPHw4K36qbiIguVV1jwcx1dSPdY/vGIEwjflG2y2FBIbt6MakVvFUKbDt+Fj9lFoqOQ0REF1m4NQf556qgD/TB325tITrOX2JBIbuKaOSHsbfV/Qv/jx8OcvE2IiIncrbchHnnn7Z8eUBr+HorBSf6aywoZHdjE2PQROuDk6VV+HTzcdFxiIjovNk/H0WZqRZxTQNtt+SdFQsK2Z2ft8r22PG8346h4FyV4ERERJRVVIYlO3IBAFPubgeFEy3KdjksKOQQ93Zsgu7Ng1FVY0HK2kOi4xARebyUHw/BYpVxZ1sdesaEiI5zVSwo5BCSJOHNe9tBkoDv9hZgR3aJ6EhERB4rNasYvxwqglLxvyUhnB0LCjlMXFMtHrolCgAwdU0mLFY+dkxE1NBqLVa8c35RtlEJUWgZFiA40bVhQSGHeimpFQJ9VDhwyojlaXmi4xAReZwlO3Jx8JQRgT4qJN/pnIuyXQ4LCjlUSIDatkrh++sPw1BZIzgREZHnOFtuwvs/1S3K9vKA1gj29xac6NqxoJDDPdKjGWLDAlBSYcbsX46IjkNE5DFmrjsMY3Ut2jUJxMiEZqLjXBcWFHI4L6UCb93bHgDw5bYTOHK6THAiIiL3tye3FMt31t1af2doeyid/LHii7GgUIPoE9sYSe10sFhlTPvuAPfpISJyIItVxpurMwEA93WNQHyzYMGJrh8LCjWY1we1g7dKgS1ZxVh/4LToOEREbmt5Wh725xugUatc5rHii7GgUIOJCvHDM7de2KfnAPfpISJygNIKM2b+VLdA5gv9WyFUoxac6MawoFCDGtcvBvpAH+SVVOGzLdmi4xARuZ331x/GucoatNZp8FhP15oY+2csKNSg6vbpqRtunLsxC6cM3KeHiMheMvINtv123h7SHiql637Nu25yclmDO4WjW7NGqKqx4F3u00NEZBdWq4w3VmdAluv+nu3Rwvn327kSFhRqcJIkYerg9pAkYHV6AXbmcJ8eIqKb9d/dJ7En9xz8vZWYMqit6Dg3jQWFhKjbpycSADD1O+7TQ0R0MwxVNZixrm5E+rk7YqEL9BGc6OaxoJAwLyW1hsZHhYx8I5al5YqOQ0Tksj7ccATF5WbEhPrjid7RouPYBQsKCRMSoMbE8/v0zFh7CGfKTIITERG5noOnjPhyWw4A4O3BcfBWucdXu3v8KchlPdazOTo01cJYXYt//HBAdBwiIpciyzLeWp0Jqwzc3UGPPrGNRUeyGxYUEkqpkDB9WAcozk+Y3XzkjOhIREQuY83eAuzIKYGvlxJTBrUTHceuWFBIuA4RWozuVXfP9PVVGVxhlojoGpRV1+CfPxwEADx7e0s0DfIVnMi+WFDIKUxMaoUmWh/kllRizsajouMQETm9ORuzUFRmQvMQP/ztVveYGPtnLCjkFALUKkwd3B4AsGDzcRw5XSY4ERGR8zp6ugyfn98u5K3B7aFWKQUnsj8WFHIaA9rr0b+dDjUWGVNW7oeVa6MQEV3CapXx6or9qLXKuLOtDv1ah4mO5BAsKORU3h7cHn7eSqTllOI/O/NExyEicjpf/3ECu06Uwt9biWlD2ouO4zAsKORUwoN8bWujpKw9hOJyro1CRHTBKUMVZqw7DACYNLANwt1sYuyfsaCQ0xndqznahwfCUPW/GepERJ5OlmW8sSoT5aZadIkKwiM9momO5FAOLygpKSmQJAnJycm2c7IsY+rUqQgPD4evry8SExORmZnp6CjkIlRKBaYP6wBJAlbuyceWo8WiIxERCbc2oxA/HzwNL6WEGfd1hFIhiY7kUA4tKGlpaViwYAE6duxY7/zMmTMxa9YszJ07F2lpadDr9ejfvz/KyvjkBtXpFBmEx3s2BwC8vmo/10YhIo9mqKzBm6vr/kP+74kt0UqnEZzI8RxWUMrLyzFq1Ch8+umnaNSoke28LMuYPXs2pkyZguHDhyMuLg6LFi1CZWUllixZ4qg45IJeTGoFXaAaOWcr8e9fs0THISISJmXtQRSXmxAT6o/x/WJEx2kQDiso48ePx6BBg3DnnXfWO5+dnY3CwkIkJSXZzqnVavTt2xepqamX/VkmkwlGo7HeQe5P4+OFqffWzVD/eNMxZBVxhI2IPM+2Y2exLK3uqcZ37+volmueXI5DCsqyZcuwe/dupKSkXPJaYWEhAECn09U7r9PpbK9dLCUlBVqt1nZERkbaPzQ5pYFxetzRJgw1Fhmvrcjg2ihE5FGqayx4beV+AMCohCjc0jxYcKKGY/eCkpeXh+effx6LFy+Gj4/PX14nSfUn98iyfMm5CyZPngyDwWA78vK4PoankCQJbw9pD18vJXbklODbXSdFRyIiajAf/XIU2cUV0AWq8cpdbUTHaVB2Lyi7du1CUVER4uPjoVKpoFKpsGnTJnz00UdQqVS2kZOLR0uKioouGVW5QK1WIzAwsN5BniOikZ9tbZTpaw/iLNdGISIPcKDAiAWbjwMApg2JQ6CPl+BEDcvuBeWOO+7A/v37kZ6ebju6deuGUaNGIT09HS1atIBer8eGDRts7zGbzdi0aRN69epl7zjkJp7o3RxtmwTiXCXXRiEi92exypi8Yh9qrTLuitNjQHu96EgNTmXvH6jRaBAXF1fvnL+/P0JCQmznk5OTMX36dMTGxiI2NhbTp0+Hn58fRo4cae845Cbq1kaJw/D5qVixJx/3dgpHvzbuuf8EEdEXqTnYe9IAjY8Kbw923+Xsr0TISrKTJk1CcnIyxo0bh27duiE/Px/r16+HRuP+z3XTjesS1QhP9q7bUvzVFftgqKwRnIiIyP7ySirx/k91y9m/dndbhAX+9XxOdybJsuxyj0UYjUZotVoYDAbOR/EwVWYLBn30O44XV2B416aY9WBn0ZGIiOxGlmU8vjANm4+cQffoYCx7ugcUbrRi7PV8f3MvHnIpvt5KvPdAJygkYMXufGw4cFp0JCIiu1mdXoDNR87AW6VAyvAOblVOrhcLCrmc+GaN8PStLQAAr63cj3OVZsGJiIhuXkmFGdO+PwAAeP6OWMSEBghOJBYLCrmkF/q3QkyoP86UmTB1DTeaJCLX9873B1BSYUYbvQbP3NZCdBzhWFDIJfl4KfHBg52hkIBV6QVYl3H5VYiJiFzBuoxCrNyTD4UEpAzvAC8lv575CZDL6hwZhDF96zbNen3VfpRU8FYPEbme4nITppxfzn5M3xh0iWp0lXd4BhYUcmnJd8ailS4AxeVmvLk6Q3QcIqLrIssyXv3vfpw9f2sn+c5Y0ZGcBgsKuTS1Son3H+gEpULC9/tO4cf9p0RHIiK6Zt/sOomfD56Gt1KBD0d09pidiq8FCwq5vI4RQRiXeOFWTwaKuVcPEbmAvJJKTPuu7qmdiUmt0LYJ1/X6MxYUcgsTbo9FG70GJRVmvLEqAy64/iAReRCrVcZL3+xFuakW3f60dAL9DwsKuQVvlQLvP9AJKoWEtRmF+H4fb/UQkfP6fGs2/sgugZ+3Eh88WHebmupjQSG3EddUi2dvbwkAeHN1Bs6U8VYPETmfI6fLMPP8XjuvD2qHZiH+ghM5JxYUcivj+7VEuyaBKK2sweur9vNWDxE5FXOtFS8sT4e51op+rUPxcPdI0ZGcFgsKuRUvZd2tHi+lhJ8yT2PN3gLRkYiIbOZsPIrMAiOC/Lww476OkCTe2vkrLCjkdtqFB+K52+vWEnhzdSaKjNWCExERAbtzS/HvX7MAAP8c2gFhgT6CEzk3FhRyS2MTY9ChqRaGqhq8+M1eWK281UNE4lSaa/Hif/bCKgNDOodjUMcmoiM5PRYUckteSgVmPdgJPl4K/H60GJ9tyRYdiYg82LtrDyG7uAL6QB9MGxwnOo5LYEEhtxWr0+DNe9oDAGb+dAj7Tp4TG4iIPNLmI2fw5bYTAID3HugIrZ+X4ESugQWF3NrD3SNxV5weNRYZzy3dg3JTrehIRORBDJU1mPTtPgDAYz2b4dbYUMGJXAcLCrk1SZLw7vCOCNf6IOdsJTcUJKIG9eaaDBQaq9GisT8m39VWdByXwoJCbk/r54V/PdwFCglYsTsfK/ecFB2JiDzAmr0FWJ1eAIUEfPBgJ/h6cyPA68GCQh7hlubBeP6OVgCA11dm4MTZCsGJiMid5RRX4LUV+wHULSDZJaqR4ESuhwWFPMazt7dE9+hgVJgteG7pHphrraIjEZEbqq6xYPyS3Sg31aJ782A8f0es6EguiQWFPIZSIWH2iM7Q+nph70kDPthwWHQkInJD0388iMwCIxr5eeFfD3eGSsmv2hvBT408SniQL2bc1xEA8Mmm49h85IzgRETkTn7cf8r2SPGsEZ3RROsrOJHrYkEhjzMwTo9RCVEAgIn/2Yvicu56TEQ3L/dsJV45/0jx2L4x6Nc6THAi18aCQh7pjXvaoZUuAMXlprrlp7kUPhHdBFNt3byTMlMt4ps1wotJrURHcnksKOSRfLyUmPNwV6hVCmw6cgafb+VS+ER041J+PIT9+QYE+XlhzsNd4MV5JzeNnyB5rNZ6DV6/px0AYMa6Q8jINwhORESuaF1GIb5IzQEAfPBAJ4QHcd6JPbCgkEd7JCEKSe10qLHImLB0Dyq4FD4RXYe8kkpM+nYvAOCZ21rgjrY6wYncBwsKeTRJkjDz/o5oovVBdnEF3lydKToSEbkIc60Vzy7dA2N1LbpEBeHlAa1FR3IrLCjk8YL8vPHhiM5QSMB/d5/E8rRc0ZGIyAXMXHcIe/POQevLeSeOwE+TCECPFiGY2L9u1v0bqzKRnndObCAicmobDpzG/22pm1z//gOdENHIT3Ai98OCQnTeuMSW6N9OB7PFir8v3sX1UYjosk6WVuKlb+rmnTzVJxr923HeiSOwoBCdp1BImPVgJ7Ro7I9Thmo8u2Q3ai3cr4eI/qfGYsWEpXtgqKpBp8ggvDKwjehIbosFhehPND5eWPBYPPy9ldh+vAQpaw+JjkRETuT9nw5jT+45aHxUmPtwF3ir+DXqKPxkiS7SMkyDDx7sBAD4bEs2VqfnC05ERM7gh32n8Mnm4wCA9+7vhMhgzjtxJBYUossYGNcEf0+MAQC88t99OHjKKDgREYmUWWCwzTt5+tZoDIzTC07k/lhQiP7CS0mtcWtsY1TXWDHmq10wVNaIjkREAhSXm/DMl7tQVWPBba1C8epdbUVH8ggsKER/QamQ8NFDXRDRyBe5JZV4fvkeWLipIJFHMddaMW7xbuSfq0J0Y3/MeagLlApJdCyPwIJCdAWN/L3x8SPxUKsU+O3wGcz++YjoSETUgN7+LhM7ckqgUavw6WPdoPXzEh3JY7CgEF1FXFMt3r2vAwBgzsYsrM8sFJyIiBrC4u0n8PUfuZAk4KOHu6BlWIDoSB6FBYXoGgzrEoHRvZoDACb+Zy+OnSkXG4iIHGr78bOYuqZub65JA9qgX5swwYk8DwsK0TWaMqgtujcPRrmpFmO+2oVy7nxM5JbySiox7uvdqLXKGNwpHGP7thAdySOxoBBdIy+lAnNHdYEuUI2sonK8/M1eyDInzRK5k0pzLZ7+cidKKsyIaxqIGfd1hCRxUqwILChE1yFM44N5o+LhpZSwNqMQ//41S3QkIrITWZbx0jd7caiwDI0D1FjwaDf4eitFx/JYLChE1ym+WSNMHdweAPD++iNcaZbITczdmIUf9xfCSynh40e6IjzIV3Qkj8aCQnQDRiU0w1N9ogEAL3+zDzuySwQnIqKbsT6zEB9sqFtG4J0hcejWPFhwImJBIbpBr93dFgPa62C2WPH0lzv5ZA+RizpcWIYXlqcDAB7v2QwPdY8SG4gAsKAQ3TClQsLsEV3QOTIIhqoajF64A8XlJtGxiOg6lFaY8fSXO1FhtqBnixC8fk870ZHoPBYUopvg663E/z3eDVHBfsgrqcLfFu1EldkiOhYRXYPqGgvGfLULuSWViAz2xbxRXeGl5Neis+D/E0Q3qXGAGgufuAVaXy+k551DMvfsIXJ6FquM5GXpdcvY+6jwf4/dgkb+3qJj0Z+woBDZQUxoAD59rBu8lQr8lHka0388KDoSEf0FWZbx1poMrMsshLdSgU8f64bWeo3oWHQRFhQiO+keHYz3HugIAPhsSza+2JotOBERXc7cjVlYvL1uj53ZD3VGjxYhoiPRZbCgENnRkM5N8fKA1gCAt78/wI0FiZzMsh25tseJp97bHnd3aCI4Ef0VFhQiOxuXGIOHu0dCloHnlu3B3rxzoiMREYCfD5zGayv3AwDG94vB4+c3ACXnxIJCZGeSJOGdIXHo2yoU1TVWPLUoDXkllaJjEXm0XSdK8ezS3bDKwP3xEXgpqbXoSHQVLChEDqBSKvDvUV3RtkkgisvNeOKLNBgqa0THIvJIWUXleGpRGqprrOjXOhQpwztwA0AXwIJC5CABahUWjr4F+kAfZBWV45mvdqK6hmukEDWk08ZqPP75DpyrrEGnyCD8m2uduAy7/7+UkpKCW265BRqNBmFhYRg6dCgOHz5c7xpZljF16lSEh4fD19cXiYmJyMzMtHcUIuH0Wh8sfOIWBKhV+CO7BM8u2Q1zrVV0LCKPYKiqweOf70D+uSq0aOyPhaNvgZ+3SnQsukZ2LyibNm3C+PHjsX37dmzYsAG1tbVISkpCRUWF7ZqZM2di1qxZmDt3LtLS0qDX69G/f3+UlZXZOw6RcG2bBGLBY/FQqxT4+WARkpfvQa2FJYXIkaprLHjmy504VFiGUI0ai57sjmAuxOZSJFmWHbrk5ZkzZxAWFoZNmzbhtttugyzLCA8PR3JyMl555RUAgMlkgk6nw4wZMzBmzJir/kyj0QitVguDwYDAwEBHxieym98OF+HpL3eixiJjWJem+OCBTlAoeB+cyN4sVhkTlu7Gj/sLoVGrsHxMT7QL53eFM7ie72+H34gzGAwAgODguq2rs7OzUVhYiKSkJNs1arUaffv2RWpqqqPjEAmT2DoMc0d2hVIhYeWefExZtR8O/u8DIo8jyzKmfZeJH/fXrRL7yWPxLCcuyqEFRZZlTJw4EX369EFcXBwAoLCwbuEqnU5X71qdTmd77WImkwlGo7HeQeSKBrTXY/aIzlBIwNIdeXj7uwMsKUR2Issy3l17CIu2nYAkAbNGdEKvmMaiY9ENcmhBefbZZ7Fv3z4sXbr0ktcufsRLluW/fOwrJSUFWq3WdkRGRjokL1FDuLdTOGbe3wkA8EVqDt5dd4glhegmXSgnn2w+DgCYNiQO93QMF5yKbobDCsqECROwZs0a/Prrr4iIiLCd1+v1AHDJaElRUdEloyoXTJ48GQaDwXbk5eU5KjZRg7g/PgL/HFY3qvjJpuOY/fNRwYmIXNfF5eSdIe3xaI9mglPRzbJ7QZFlGc8++yxWrFiBjRs3Ijo6ut7r0dHR0Ov12LBhg+2c2WzGpk2b0KtXr8v+TLVajcDAwHoHkasbldAMb9zTDgDwr1+OYv5vxwQnInI9ly0nPZuLDUV2YfcHwsePH48lS5Zg9erV0Gg0tpESrVYLX19fSJKE5ORkTJ8+HbGxsYiNjcX06dPh5+eHkSNH2jsOkVN7qk80qmsseO+nw5ix7hB8vBR4onf01d9IRCwnbs7uBWX+/PkAgMTExHrnFy5ciNGjRwMAJk2ahKqqKowbNw6lpaVISEjA+vXrodFo7B2HyOmN79cSphoLPtqYhbe/OwC1SomRCVGiYxE5NZYT9+fwdVAcgeugkLuRZRkpaw9hwebjkCTggwc6YXjXiKu/kcgDsZy4LqdaB4WIrk6SJEy+qw0e69kMsgy89M1efLe3QHQsIqfDcuI5WFCInIQkSZh6b3uM6BYJqww8v2wPlu7IFR2LyGmwnHgWFhQiJ6JQSJg+vAMe7l5XUiav2M+ne4jAcuKJWFCInIxSIWH6sA74e2IMAGDGukNI+fEgF3Mjj8Vy4plYUIickCRJeGVgG7x2dxsAwCebj+OV/+7jLsjkcaxWGdO+P8By4oFYUIic2DO3xWDm/R2hkID/7DyJ8Ut2o7rGIjoWUYOorrFgwrI9WLg1BwDLiadhQSFycg92i8S8UfHwVirwU+ZpPPlFGspNtaJjETmUobIGj32+Az/sOwUvpYR/PdSZ5cTDsKAQuYCBcXp88eQt8PdWIvXYWYz8dDvOlptExyJyiPxzVbj/41TsyC6BRq3Coie6Y0jnpqJjUQNjQSFyEb1iGmPpMz0Q7O+NfScNeOCTbSg4VyU6FpFdHSgwYvi8rThaVA59oA/+M7YnerVsLDoWCcCCQuRCOkYE4ZuxPRGu9cHxMxW4f34qsorKRccisoutWcV48JNtOG00oZUuACvG9ULbJlwt3FOxoBC5mJjQAHz7916ICfVHgaEaD36yDftOnhMdi+imrE7Px+iFO1BuqkVCdDC+GdsL4UG+omORQCwoRC4oPMgX34zthY4RWpRUmPHwgu349VCR6FhE102WZcz/7RieX5aOGouMQR2b4MunukPr6yU6GgnGgkLkooL9vbHk6R7oFROCCrMFTy5Kw7zfsrigG7kMi1XGW2syMWPdIQDA3/pEY85DXaBWKQUnI2fAgkLkwgLUKnzxRHeMSoiCLAMz1x3GhKV7UGXmWink3KprLBj39S58ue0EJAl44552eP2edlAoJNHRyEmwoBC5OG+VAv8c1gH/GBoHlULC9/tO4b75qThZWik6GtFlnSkzYdT//YGfMk/DW6XA3Ie74qk+0aJjkZNhQSFyE4/0aIYlT/dAiL83DpwyYvDcrdh+/KzoWET17MwpwaCPfseuE6UI9FFh8VMJGNSxiehY5IRYUIjcSPfoYKyZ0AdxTQNRUmHGI//3B77alsN5KSScLMv4bEs2HlqwHUVlJrQMC8CKcb3RPTpYdDRyUiwoRG6maZAvvhnTC0M6h6PWKuON1ZmYvGI/TLWcl0JilJtq8eySPXjn+wOotcoY3Ckcq8f3RsuwANHRyImpRAcgIvvz9VZi9ojOaNckEO+uO4RlaXk4WlSO+Y90RZjGR3Q88iBHT5dhzOJdOH6mAl5KCa8PaofHejaDJHEyLF0ZR1CI3JQkSRjTNwYLR98CjY8Ku06UYvCcrdibd050NPIQq9PzMeTfW3H8TAX0gT5Y9kxPPN6rOcsJXRMWFCI3l9g6DKvH90ZMqD8KjdV44JNt+M/OPM5LIYcx11oxdU0mnl+WjkqzBb1bhuCH5/ogvlkj0dHIhbCgEHmAFqEBWDW+N+5sGwZzrRWTvt2HZ5fsQWmFWXQ0cjOnDFUYsWAbvkjNAQA8268lvnwyASEBarHByOWwoBB5CI2PFxY82g0vJbWCSiHhh/2nMGD2Zmw+ckZ0NHITW44WY9BHW7An9xwCfVT47PFueGlAayi5+BrdAEl2wXFeo9EIrVYLg8GAwEDudEl0vfadPIfk5ek4fqYCADC6V3O8elcb+HhxiXG6fjUWK/79axY++uUorDLQPjwQ80fFIyrET3Q0cjLX8/3NgkLkoarMFqSsPYgvt50AAMSE+uNfD3VBXFOt4GTkSjILDHj5m304cMoIABjRLRJvD2nPskuXxYJCRNfst8NFmPTtPhSVmaBSSHihfyuM7RvDYXm6InOtFXM3HsW8346h1iojyM8Lbw9ujyGdm4qORk6MBYWIrktphRmvrdyPtRmFAID4Zo3w4YOdOURPl7Xv5Dm8/M0+HD5dBgC4K06PaUPiEKrhRFi6MhYUIrpusizjv7vzMXVNJspNtfD3VuKte9vjgW4RXLeCANTtQPyvX45iwebjsFhlhPh7452hcbi7A/fSoWvDgkJENyyvpBIv/mcvduSUAACS2unwj6FxCAvkCrSebHduKSZ9uw9ZReUAgMGdwjF1cHsE+3sLTkauhAWFiG6KxSrj09+P44P1h1FjkeHvrcSEO2LxRO/mUKs4+dGTVNdY8MH6w/hsSzasMhCqUeOfQ+OQ1F4vOhq5IBYUIrKLzAIDpqzMQPr55fGjG/vjzXvaoV+bMLHBqEGk5ZRg0rf7kF1c9zj68K5N8eY97RDkx1ETujEsKERkN1arjBV78vHu2kMoLjcBAPq1DsUb97RDi1DuRuuOTpZWYtb6I1iZng9ZBnSBaqQM74Db2+hERyMXx4JCRHZXVl2DuRuz8PnWbNRYZHgpJTzZJxoTbo9FgJobo7sDQ2UN5v2WhYWpOTDXWgHUrWvy2qC20Pp6CU5H7oAFhYgc5viZckz7/gB+O1y3RH6oRo1XB7bBsC5NoeDaKS7JVGvBV9tOYM7GLBiqagAAPVoE47W726JjRJDYcORWWFCIyOE2HjqNad8dQM7ZSgBA58ggvD24PTpFBokNRtfMapXx3b4CvPfTYZwsrQIAtNIFYPJdbZHYOpSPl5PdsaAQUYMw1VqwcGsO5vxyFBVmC4C6iZTj+7VEDOenOLXUrGJMX3sQGfl1S9TrAtV4Mak17usawVWEyWFYUIioQZ02VmPGukNYsTsfACBJwMD2eoxLbIkOEdzbx5kcKjTi3bWHbLfoAtQq/D0xBk/2joavNx8hJ8diQSEiIdLzzmHuxiz8fPC07Vyflo0xLjEGPWNCeMtAoIx8Az7bko3V6fmwyoBKIeGRHs0w4faWCAngEvXUMFhQiEioI6fL8PFvx7B6bwEs1rq/YjpFBuHvfWOQ1E7HybQNpNZixfoDp7FwazbSckpt5wd1aIKXB7RG88b+AtORJ2JBISKnkFdSiU9/P47laXkwnX9stWVYAMb2jcGQzuHwUioEJ3RP5yrNWJaWh6+2nUD+ubrJryqFhEEdm+CpPtF8MoeEYUEhIqdypsyEL1Kz8eW2EyirrgUAhGt98PRtLXB/fAQ0Plxjwx6Oni7DwtQcrNh9EtU1dYUw2N8boxKi8EiPZtBxPyUSjAWFiJySsboGX2/PxWdbsm2r0qpVCtzZTodhnZvitlah8FZxVOV6WK0yfjtShIVbc/D70WLb+bZNAvFE7+YY3CkcPl6c/ErOgQWFiJxadY0F3+46iYVbs3HsTIXtfCM/Lwzq2ATDujRF16hGnFT7F2RZxpHT5VibcQqr0wtse+UoJKB/Ox2e6B2NhOhgfn7kdFhQiMglyLKMjHwjVqXnY3V6gW1UBQCigv0wpHM4hnRuipZhXFPFapWxL9+AdRmF+Cmz0FZKAEDjo8KIbpF4vFdzRAb7CUxJdGUsKETkcmotVqQeO4tVe/KxLrMQlecXfgOAjhFaDOncFPd0bOJR8yhqLVak5ZTip8y6UnLKUG17zVulwG2xoRjQXoe7OzSBP/dDIhfAgkJELq3SXIsNB05jdXoBNh05Y3tUGQBahPojIToEPVoEIyE6BHqtexUWU60FqVlnsS6jEBsOnkZJhdn2mr+3Ev3ahGFgnB6JrcO4SSO5HBYUInIbxeUm/LDvFFal52NP7rlLXm8W4ofuzYOR0CIECdHBLnWLw2qVcby4HHvzDNh38hz2njTgwCmjbSdhAAjy80L/tjoMjNOjd8vGnPBKLo0FhYjc0rlKM3Zkl+CP7BLsyC5BZoEB1ov+Bmsa5Ivu0cFIiA5Ga70GUcF+CPb3Fj5hVJZl5J+rwr6TBuw9eQ778gzIyDegzFR7ybW6QDUGtNdjYHs9ukcHQ8X1YshNsKAQkUcwVtdgV04ptmefxY7sEuw/aUDtxY0FdbdGIoP9EBXsZ/vfC/8c0cjXLqMSNRYristNOFN20VFuQl5JJfadNODsn27XXODjpUBcuBYdI4LQKVKLThFBaBbiJ7xQETkCCwoReaQKUy1255bij+Ml2HmiBDnFlSg0Vl/1fbpANfRaX3grJagUCqiUElQKCSqlAl4XzimkuvPKun8uq66tV0JKLlM+LqZSSGjTRFNXRiLqSklsWABHSMhjXM/3N2dYEZHb8FercGtsKG6NDbWdq66xIP9cFXJLKpFXUoncs5XIK61EbkkVcs9WoMJswWmjCaeNpiv85GujVEhoHOCNMI0PQjVqhAaoEapRQ6f1QVx4INo2CeQcEqJrxIJCRG7Nx0uJmNAAxIReupaKLMsoraxBbkklzpSZUGuxosYqo9ZiRa1FRq1VRq3VihrL+XNWGTXnX9P4qOpKyIUjQI1Gft7cCJHITlhQiMhjSZKEYH9vBPt7i45CRBfhjU8iIiJyOiwoRERE5HRYUIiIiMjpsKAQERGR02FBISIiIqfDgkJEREROR2hBmTdvHqKjo+Hj44P4+Hj8/vvvIuMQERGRkxBWUJYvX47k5GRMmTIFe/bswa233oq77roLubm5oiIRERGRkxC2F09CQgK6du2K+fPn2861bdsWQ4cORUpKyhXfy714iIiIXM/1fH8LGUExm83YtWsXkpKS6p1PSkpCamrqJdebTCYYjcZ6BxEREbkvIQWluLgYFosFOp2u3nmdTofCwsJLrk9JSYFWq7UdkZGRDRWViIiIBBA6SVaS6m+qJcvyJecAYPLkyTAYDLYjLy+voSISERGRAEI2C2zcuDGUSuUloyVFRUWXjKoAgFqthlqtbqh4REREJJiQguLt7Y34+Hhs2LABw4YNs53fsGEDhgwZctX3X5jXy7koREREruPC9/a1PJ8jpKAAwMSJE/Hoo4+iW7du6NmzJxYsWIDc3FyMHTv2qu8tKysDAM5FISIickFlZWXQarVXvEZYQRkxYgTOnj2LadOm4dSpU4iLi8OPP/6IZs2aXfW94eHhyMvLg0ajueycFU9kNBoRGRmJvLw8PnrdQPiZNyx+3g2Pn3nD8oTPW5ZllJWVITw8/KrXClsHheyLa8M0PH7mDYufd8PjZ96w+HnXx714iIiIyOmwoBAREZHTYUFxE2q1Gm+99RYfx25A/MwbFj/vhsfPvGHx866Pc1CIiIjI6XAEhYiIiJwOCwoRERE5HRYUIiIicjosKEREROR0WFDcmMlkQufOnSFJEtLT00XHcVs5OTl46qmnEB0dDV9fX8TExOCtt96C2WwWHc2tzJs3D9HR0fDx8UF8fDx+//130ZHcUkpKCm655RZoNBqEhYVh6NChOHz4sOhYHiMlJQWSJCE5OVl0FOFYUNzYpEmTrmk5Ybo5hw4dgtVqxSeffILMzEx8+OGH+Pjjj/Haa6+JjuY2li9fjuTkZEyZMgV79uzBrbfeirvuugu5ubmio7mdTZs2Yfz48di+fTs2bNiA2tpaJCUloaKiQnQ0t5eWloYFCxagY8eOoqM4BT5m7KbWrl2LiRMn4r///S/at2+PPXv2oHPnzqJjeYz33nsP8+fPx/Hjx0VHcQsJCQno2rUr5s+fbzvXtm1bDB06FCkpKQKTub8zZ84gLCwMmzZtwm233SY6jtsqLy9H165dMW/ePPzjH/9A586dMXv2bNGxhOIIihs6ffo0nn76aXz11Vfw8/MTHccjGQwGBAcHi47hFsxmM3bt2oWkpKR655OSkpCamioolecwGAwAwH+fHWz8+PEYNGgQ7rzzTtFRnIaw3YzJMWRZxujRozF27Fh069YNOTk5oiN5nGPHjmHOnDn44IMPREdxC8XFxbBYLNDpdPXO63Q6FBYWCkrlGWRZxsSJE9GnTx/ExcWJjuO2li1bht27dyMtLU10FKfCERQXMXXqVEiSdMVj586dmDNnDoxGIyZPniw6ssu71s/8zwoKCjBw4EA88MAD+Nvf/iYouXuSJKner2VZvuQc2dezzz6Lffv2YenSpaKjuK28vDw8//zzWLx4MXx8fETHcSqcg+IiiouLUVxcfMVrmjdvjoceegjfffddvb+4LRYLlEolRo0ahUWLFjk6qtu41s/8wl8qBQUF6NevHxISEvDFF19AoWD/twez2Qw/Pz988803GDZsmO38888/j/T0dGzatElgOvc1YcIErFq1Cps3b0Z0dLToOG5r1apVGDZsGJRKpe2cxWKBJElQKBQwmUz1XvMkLChuJjc3F0aj0fbrgoICDBgwAN9++y0SEhIQEREhMJ37ys/PR79+/RAfH4/Fixd77F8ojpKQkID4+HjMmzfPdq5du3YYMmQIJ8namSzLmDBhAlauXInffvsNsbGxoiO5tbKyMpw4caLeuSeeeAJt2rTBK6+84tG31jgHxc1ERUXV+3VAQAAAICYmhuXEQQoKCpCYmIioqCi8//77OHPmjO01vV4vMJn7mDhxIh599FF069YNPXv2xIIFC5Cbm4uxY8eKjuZ2xo8fjyVLlmD16tXQaDS2eT5arRa+vr6C07kfjUZzSQnx9/dHSEiIR5cTgAWF6KatX78eWVlZyMrKuqQEcoDSPkaMGIGzZ89i2rRpOHXqFOLi4vDjjz+iWbNmoqO5nQuPcicmJtY7v3DhQowePbrhA5HH4i0eIiIicjqcxUdEREROhwWFiIiInA4LChERETkdFhQiIiJyOiwoRERE5HRYUIiIiMjpsKAQERGR02FBISIiIqfDgkJEREROhwWFiIiInA4LChERETkdFhQiIiJyOv8PJ6lZvsMddPkAAAAASUVORK5CYII=\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(xs, ys)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative on increasing side of the curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14.000001158365194</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m14.000001158365194\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0000000001\n",
    "x = 3.0\n",
    "(f(x + h) - f(x))/h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative on the decreasing side of the curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-21.999966293151374</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-21.999966293151374\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0000000001\n",
    "x = -3.0\n",
    "(f(x + h) - f(x))/h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative on the bottom of the curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0000000001\n",
    "x = 2/3\n",
    "(f(x + h) - f(x))/h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multivariable: `a`, `b`, `c` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m4.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "def d(a, b, c): return a*b + c\n",
    "print(d(a, b, c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative with respect to `a`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d1 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d1 \u001b[1;36m4.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d2 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.999699999999999</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d2 \u001b[1;36m3.999699999999999\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">slope <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.000000000010772</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "slope \u001b[1;36m-3.000000000010772\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0001\n",
    "\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = d(a, b, c)\n",
    "\n",
    "a += h\n",
    "d2 = d(a, b, c)\n",
    "\n",
    "\n",
    "print('d1', d1)\n",
    "print('d2', d2)\n",
    "print('slope', (d2 - d1)/h)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative with respect to `b` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d1 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d1 \u001b[1;36m4.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d2 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0002</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d2 \u001b[1;36m4.0002\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">slope <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0000000000042206</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "slope \u001b[1;36m2.0000000000042206\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0001\n",
    "\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = d(a, b, c)\n",
    "\n",
    "b += h\n",
    "d2 = d(a, b, c)\n",
    "\n",
    "\n",
    "print('d1', d1)\n",
    "print('d2', d2)\n",
    "print('slope', (d2 - d1)/h)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Derivative with respect to `c`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d1 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d1 \u001b[1;36m4.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">d2 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0001</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "d2 \u001b[1;36m4.0001\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">slope <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9999999999976694</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "slope \u001b[1;36m0.9999999999976694\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 0.0001\n",
    "\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = d(a, b, c)\n",
    "\n",
    "c += h\n",
    "d2 = d(a, b, c)\n",
    "\n",
    "\n",
    "print('d1', d1)\n",
    "print('d2', d2)\n",
    "print('slope', (d2 - d1)/h)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Value Object\n",
    "(mentioned in the README of [micrograd](https://github.com/karpathy/micrograd) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define intial template of Value Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data):\n",
    "        self.data = data\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span><span style=\"font-weight: bold\">)</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.0</span><span style=\"font-weight: bold\">))</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m2\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m, \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-3.0\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(-3.0)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the `add` function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data):\n",
    "        self.data = data\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\"\n",
    "\n",
    "    def __add__(self, other): # ⭠ for adding among the value objects\n",
    "        return Value(self.data + other.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span><span style=\"font-weight: bold\">)</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.0</span><span style=\"font-weight: bold\">))</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m2\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m, \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-3.0\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(-3.0)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-1.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a + b # a.__add__(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the `mul` function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data):\n",
    "        self.data = data\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data)\n",
    "\n",
    "    def __mul__(self, other): # ⭠ for multiplying among the value objects\n",
    "        return Value(self.data * other.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span><span style=\"font-weight: bold\">)</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.0</span><span style=\"font-weight: bold\">))</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m2\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m, \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-3.0\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(-3.0)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-6.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-6.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a * b # a.__mul__(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = Value(10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m4\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "d = a * b + c; d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the functionality to know what values created a value with `_children`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=()): # ⭠ Add _children\n",
    "        self.data = data\n",
    "        self._prev = set(_children) # ⭠ Add _children\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data ={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data, (self, other))\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        return Value(self.data * other.data, (self, other))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span>data =<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0mdata =\u001b[1;36m4.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(-3.0)\n",
    "c = Value(10.0)\n",
    "d = a*b + c\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span>data =<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-6.0</span><span style=\"font-weight: bold\">)</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span>data =<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10.0</span><span style=\"font-weight: bold\">)}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0mdata =\u001b[1;36m-6.0\u001b[0m\u001b[1m)\u001b[0m, \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0mdata =\u001b[1;36m10.0\u001b[0m\u001b[1m)\u001b[0m\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "d._prev # childrens are -6.0 (a *b) and 10.0 (c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the functionality to know what operations created a value with `_op`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op=''): # ⭠ Add _op\n",
    "        self.data = data\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op # ⭠ Add _op\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        return Value(self.data * other.data, (self, other), '*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m4\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(-3.0)\n",
    "c = Value(10.0)\n",
    "d = a*b + c\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10.0</span><span style=\"font-weight: bold\">)</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-6.0</span><span style=\"font-weight: bold\">)}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m10\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m, \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdata\u001b[0m=\u001b[1;36m-6.0\u001b[0m\u001b[1m)\u001b[0m\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "d._prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000\">'+'</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[32m'+'\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "d._op"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the expression graph with operators and operands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "    # build a set of all nodes and edges in a graph\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for child in v._prev:\n",
    "                edges.add((child, v))\n",
    "                build(child)\n",
    "    build(root)\n",
    "    return nodes, edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_dot(root, label):\n",
    "    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
    "\n",
    "    nodes, edges = trace(root)\n",
    "    for n in nodes:\n",
    "        uid = str(id(n))\n",
    "        # for any value in the graph, create a rectangular ('record') node for it\n",
    "        dot.node(name = uid, label=label(n), shape='record') # ⭠ label function getting called\n",
    "        if n._op:\n",
    "            # if this value is a result of some operation, create an op node for it\n",
    "            dot.node(name = uid + n._op, label = n._op)\n",
    "            dot.edge(uid + n._op, uid)\n",
    "  \n",
    "    for n1, n2 in edges:\n",
    "        # connect n1 to the op node of n2\n",
    "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "  \n",
    "    return dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"512pt\" height=\"127pt\"\n",
       " viewBox=\"0.00 0.00 512.00 127.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 123)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-123 508,-123 508,4 -4,4\"/>\n",
       "<!-- 140404033556048 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404033556048</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"212,-27.5 212,-63.5 297,-63.5 297,-27.5 212,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"254.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404041162192+ -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404041162192+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"361\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"361\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404033556048&#45;&gt;140404041162192+ -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404033556048&#45;&gt;140404041162192+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M297.1,-56.25C306.61,-58.71 316.62,-61.29 325.78,-63.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"325.13,-67.11 335.69,-66.22 326.88,-60.33 325.13,-67.11\"/>\n",
       "</g>\n",
       "<!-- 140404033556048* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404033556048*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"148\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404033556048*&#45;&gt;140404033556048 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404033556048*&#45;&gt;140404033556048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M175.1,-45.5C183.27,-45.5 192.59,-45.5 201.88,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"201.89,-49 211.89,-45.5 201.89,-42 201.89,-49\"/>\n",
       "</g>\n",
       "<!-- 140404038599760 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404038599760</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"211,-82.5 211,-118.5 298,-118.5 298,-82.5 211,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"254.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "</g>\n",
       "<!-- 140404038599760&#45;&gt;140404041162192+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404038599760&#45;&gt;140404041162192+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M298,-89.11C307.18,-86.65 316.78,-84.08 325.6,-81.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"326.78,-85.02 335.53,-79.06 324.97,-78.26 326.78,-85.02\"/>\n",
       "</g>\n",
       "<!-- 140404041162384 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404041162384</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 82.5,-91.5 82.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"42.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "</g>\n",
       "<!-- 140404041162384&#45;&gt;140404033556048* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404041162384&#45;&gt;140404033556048*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M82.63,-62.91C92.66,-60.2 103.36,-57.31 113.09,-54.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"114.09,-58.03 122.83,-52.04 112.26,-51.27 114.09,-58.03\"/>\n",
       "</g>\n",
       "<!-- 140404041160976 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404041160976</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 85,-36.5 85,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"42.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 140404041160976&#45;&gt;140404033556048* -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404041160976&#45;&gt;140404033556048*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M85,-29.33C94.14,-31.71 103.72,-34.21 112.54,-36.51\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"111.91,-39.96 122.47,-39.1 113.68,-33.19 111.91,-39.96\"/>\n",
       "</g>\n",
       "<!-- 140404041162192 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404041162192</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"424,-54.5 424,-90.5 504,-90.5 504,-54.5 424,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"464\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404041162192+&#45;&gt;140404041162192 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404041162192+&#45;&gt;140404041162192</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M388.01,-72.5C395.82,-72.5 404.67,-72.5 413.48,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"413.73,-76 423.73,-72.5 413.73,-69 413.73,-76\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def label(node): return \"{data %.4f}\" % (node.data)\n",
    "draw_dot(d, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add label to each node\n",
    "> so that we know what are the corresponding variables for each value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''): # ⭠ Add label\n",
    "        self.data = data\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label # ⭠ Add label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        return Value(self.data * other.data, (self, other), '*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">L</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-8.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mL\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m-8.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0, label = 'a')\n",
    "b = Value(-3.0, label='b')\n",
    "c = Value(10, label = 'c')\n",
    "e = a*b; e.label = 'e'\n",
    "d = e + c; d.label = 'd'\n",
    "f = Value(-2.0, label='f')\n",
    "L = d * f; L.label = 'L'\n",
    "L"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change the `label` function to render the label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label(node): return \"{%s | {data %.4f}}\" % (node.label, node.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"820pt\" height=\"156pt\"\n",
       " viewBox=\"0.00 0.00 820.00 156.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 152)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-152 816,-152 816,4 -4,4\"/>\n",
       "<!-- 140404042251280 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042251280</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-111.5 2.5,-147.5 105.5,-147.5 105.5,-111.5 2.5,-111.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-111.5 25.5,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251536* -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042251536*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"171\" cy=\"-101.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"171\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042251280&#45;&gt;140404042251536* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042251280&#45;&gt;140404042251536*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M105.78,-117.14C115.85,-114.68 126.2,-112.16 135.58,-109.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"136.41,-113.28 145.3,-107.51 134.76,-106.48 136.41,-113.28\"/>\n",
       "</g>\n",
       "<!-- 140404042251344 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042251344</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"234,-28.5 234,-64.5 344,-64.5 344,-28.5 234,-28.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"245.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"257,-28.5 257,-64.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"300.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251728+ -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404042251728+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"407\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"407\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042251344&#45;&gt;140404042251728+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404042251344&#45;&gt;140404042251728+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344.26,-59.13C353.49,-61.28 362.84,-63.45 371.37,-65.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370.79,-68.9 381.32,-67.76 372.38,-62.08 370.79,-68.9\"/>\n",
       "</g>\n",
       "<!-- 140404042251408 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042251408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"470,-0.5 470,-36.5 576,-36.5 576,-0.5 470,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"480.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"491,-0.5 491,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"533.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251664* -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042251664*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"639\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"639\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042251408&#45;&gt;140404042251664* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042251408&#45;&gt;140404042251664*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M576.33,-30.89C585.48,-33.06 594.8,-35.27 603.33,-37.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"602.73,-40.74 613.27,-39.64 604.34,-33.93 602.73,-40.74\"/>\n",
       "</g>\n",
       "<!-- 140404042251216 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042251216</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-56.5 0,-92.5 108,-92.5 108,-56.5 0,-56.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-70.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-56.5 23,-92.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-70.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251216&#45;&gt;140404042251536* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042251216&#45;&gt;140404042251536*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M108.12,-86.97C117.33,-89.13 126.68,-91.33 135.24,-93.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"134.68,-96.8 145.21,-95.68 136.28,-89.99 134.68,-96.8\"/>\n",
       "</g>\n",
       "<!-- 140404042251536 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042251536</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"235,-83.5 235,-119.5 343,-119.5 343,-83.5 235,-83.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"246.5\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"258,-83.5 258,-119.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"300.5\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251536&#45;&gt;140404042251728+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042251536&#45;&gt;140404042251728+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M343.25,-88.65C352.84,-86.33 362.6,-83.98 371.48,-81.83\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"372.54,-85.18 381.43,-79.43 370.89,-78.37 372.54,-85.18\"/>\n",
       "</g>\n",
       "<!-- 140404042251536*&#45;&gt;140404042251536 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042251536*&#45;&gt;140404042251536</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.03,-101.5C206.08,-101.5 215.32,-101.5 224.74,-101.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"224.99,-105 234.99,-101.5 224.99,-98 224.99,-105\"/>\n",
       "</g>\n",
       "<!-- 140404042251664 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042251664</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"702,-27.5 702,-63.5 812,-63.5 812,-27.5 702,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"714.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"727,-27.5 727,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"769.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251664*&#45;&gt;140404042251664 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042251664*&#45;&gt;140404042251664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M666.03,-45.5C673.7,-45.5 682.45,-45.5 691.42,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"691.58,-49 701.58,-45.5 691.58,-42 691.58,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042251728 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042251728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"471.5,-55.5 471.5,-91.5 574.5,-91.5 574.5,-55.5 471.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"483\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"494.5,-55.5 494.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"534.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042251728&#45;&gt;140404042251664* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042251728&#45;&gt;140404042251664*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M574.67,-61.06C584.43,-58.66 594.45,-56.2 603.55,-53.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"604.5,-57.33 613.37,-51.55 602.83,-50.54 604.5,-57.33\"/>\n",
       "</g>\n",
       "<!-- 140404042251728+&#45;&gt;140404042251728 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042251728+&#45;&gt;140404042251728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M434.15,-73.5C442.26,-73.5 451.57,-73.5 461.02,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"461.28,-77 471.28,-73.5 461.28,-70 461.28,-77\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add `grad` to `Value` class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        return Value(self.data * other.data, (self, other), '*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0, label = 'a')\n",
    "b = Value(-3.0, label='b')\n",
    "c = Value(10, label = 'c')\n",
    "e = a*b; e.label = 'e'\n",
    "d = e + c; d.label = 'd'\n",
    "f = Value(-2.0, label='f')\n",
    "L = d * f; L.label = 'L'\n",
    "L.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label(node): return \"{%s | {data %.4f} | grad %.4f}\" % (node.label, node.data, node.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1148pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1148.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1144,-150 1144,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 507,-63.5 507,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M507.05,-61.67C516.78,-63.35 526.18,-64.96 534.62,-66.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"534.3,-69.91 544.75,-68.16 535.49,-63.01 534.3,-69.91\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.28,-45.5C288.05,-45.5 297.08,-45.5 306.68,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.88,-49 316.88,-45.5 306.88,-42 306.88,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"948,-81.5 948,-117.5 1140,-117.5 1140,-81.5 948,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"960.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-81.5 973,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-81.5 1058,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"885\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"885\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M912.28,-99.5C919.78,-99.5 928.44,-99.5 937.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"937.87,-103 947.87,-99.5 937.87,-96 937.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"635.5,-54.5 635.5,-90.5 820.5,-90.5 820.5,-54.5 635.5,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"647\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-54.5 658.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-54.5 738.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.51,-88.44C830.48,-90.18 840.13,-91.86 848.77,-93.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"848.31,-96.84 858.77,-95.1 849.52,-89.94 848.31,-96.84\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.29,-72.5C606.26,-72.5 615.54,-72.5 625.39,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.41,-76 635.41,-72.5 625.41,-69 625.41,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-109.5 634,-145.5 822,-145.5 822,-109.5 634,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"644.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"655,-109.5 655,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"697.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"740,-109.5 740,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M822.29,-110.65C831.57,-108.97 840.52,-107.35 848.61,-105.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"849.46,-109.3 858.68,-104.07 848.21,-102.41 849.46,-109.3\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 508,-118.5 508,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M508.4,-83.49C517.69,-81.83 526.64,-80.23 534.71,-78.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"535.53,-82.2 544.76,-77 534.31,-75.31 535.53,-82.2\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a function `lol` \n",
    "#### Derive with respect to `a`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.000000000021544</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m6.000000000021544\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.0001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0 + h, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derive with respect to `L`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9999999999976694</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.9999999999976694\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.0001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data + h\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "L.grad = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1148pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1148.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1144,-150 1144,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 507,-63.5 507,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M507.05,-61.67C516.78,-63.35 526.18,-64.96 534.62,-66.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"534.3,-69.91 544.75,-68.16 535.49,-63.01 534.3,-69.91\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.28,-45.5C288.05,-45.5 297.08,-45.5 306.68,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.88,-49 316.88,-45.5 306.88,-42 306.88,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"948,-81.5 948,-117.5 1140,-117.5 1140,-81.5 948,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"960.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-81.5 973,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-81.5 1058,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"885\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"885\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M912.28,-99.5C919.78,-99.5 928.44,-99.5 937.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"937.87,-103 947.87,-99.5 937.87,-96 937.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"635.5,-54.5 635.5,-90.5 820.5,-90.5 820.5,-54.5 635.5,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"647\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-54.5 658.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-54.5 738.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.51,-88.44C830.48,-90.18 840.13,-91.86 848.77,-93.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"848.31,-96.84 858.77,-95.1 849.52,-89.94 848.31,-96.84\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.29,-72.5C606.26,-72.5 615.54,-72.5 625.39,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.41,-76 635.41,-72.5 625.41,-69 625.41,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-109.5 634,-145.5 822,-145.5 822,-109.5 634,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"644.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"655,-109.5 655,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"697.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"740,-109.5 740,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M822.29,-110.65C831.57,-108.97 840.52,-107.35 848.61,-105.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"849.46,-109.3 858.68,-104.07 848.21,-102.41 849.46,-109.3\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 508,-118.5 508,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M508.4,-83.49C517.69,-81.83 526.64,-80.23 534.71,-78.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"535.53,-82.2 544.76,-77 534.31,-75.31 535.53,-82.2\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#### Derivative of `L` with respect to `f`\n",
    "\n",
    "$$ L = f \\cdot d $$\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial f} = \\frac{\\partial (f \\cdot d)}{\\partial f} = d = 4.0 $$\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.9999999999995595</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m3.9999999999995595\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0 + h, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.grad = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1148pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1148.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1144,-150 1144,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 507,-63.5 507,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M507.05,-61.67C516.78,-63.35 526.18,-64.96 534.62,-66.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"534.3,-69.91 544.75,-68.16 535.49,-63.01 534.3,-69.91\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.28,-45.5C288.05,-45.5 297.08,-45.5 306.68,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.88,-49 316.88,-45.5 306.88,-42 306.88,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"948,-81.5 948,-117.5 1140,-117.5 1140,-81.5 948,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"960.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-81.5 973,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-81.5 1058,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"885\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"885\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M912.28,-99.5C919.78,-99.5 928.44,-99.5 937.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"937.87,-103 947.87,-99.5 937.87,-96 937.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"635.5,-54.5 635.5,-90.5 820.5,-90.5 820.5,-54.5 635.5,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"647\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-54.5 658.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-54.5 738.5,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.51,-88.44C830.48,-90.18 840.13,-91.86 848.77,-93.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"848.31,-96.84 858.77,-95.1 849.52,-89.94 848.31,-96.84\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.29,-72.5C606.26,-72.5 615.54,-72.5 625.39,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.41,-76 635.41,-72.5 625.41,-69 625.41,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-109.5 634,-145.5 822,-145.5 822,-109.5 634,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"644.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"655,-109.5 655,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"697.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"740,-109.5 740,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M822.29,-110.65C831.57,-108.97 840.52,-107.35 848.61,-105.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"849.46,-109.3 858.68,-104.07 848.21,-102.41 849.46,-109.3\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 508,-118.5 508,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M508.4,-83.49C517.69,-81.83 526.64,-80.23 534.71,-78.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"535.53,-82.2 544.76,-77 534.31,-75.31 535.53,-82.2\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derivative of `L` with respect to `d`\n",
    "\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial d} = \\frac{\\partial (f \\cdot d)}{\\partial d} = f = -2.0 $$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.000000000000668</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-2.000000000000668\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    d.data += h\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d.grad = -2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1149pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1149.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1145,-150 1145,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 507,-63.5 507,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M507.05,-61.67C516.78,-63.35 526.18,-64.96 534.62,-66.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"534.3,-69.91 544.75,-68.16 535.49,-63.01 534.3,-69.91\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.28,-45.5C288.05,-45.5 297.08,-45.5 306.68,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.88,-49 316.88,-45.5 306.88,-42 306.88,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"949,-81.5 949,-117.5 1141,-117.5 1141,-81.5 949,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"961.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"974,-81.5 974,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1016.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1059,-81.5 1059,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1100\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"886\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M913.28,-99.5C920.78,-99.5 929.44,-99.5 938.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"938.87,-103 948.87,-99.5 938.87,-96 938.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-54.5 634,-90.5 823,-90.5 823,-54.5 634,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"645.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"657,-54.5 657,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"697\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"737,-54.5 737,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"780\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M823.1,-88.75C832.4,-90.37 841.38,-91.93 849.49,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"849.14,-96.83 859.59,-95.09 850.34,-89.93 849.14,-96.83\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.03,-72.5C605.66,-72.5 614.52,-72.5 623.94,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.94,-76 633.94,-72.5 623.94,-69 623.94,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634.5,-109.5 634.5,-145.5 822.5,-145.5 822.5,-109.5 634.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"645\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"655.5,-109.5 655.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"740.5,-109.5 740.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"781.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M822.65,-110.73C832.2,-109.01 841.43,-107.35 849.72,-105.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"850.47,-109.27 859.69,-104.06 849.23,-102.39 850.47,-109.27\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 508,-118.5 508,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M508.4,-83.49C517.69,-81.83 526.64,-80.23 534.71,-78.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"535.53,-82.2 544.76,-77 534.31,-75.31 535.53,-82.2\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derivative of `L` with respect to `c`\n",
    "\n",
    "$$ \\frac{\\partial d}{\\partial c} = \\frac{\\partial (c + e)}{\\partial c} = 1.0 $$\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial c} = \\frac{\\partial L}{\\partial d}\\cdot\\frac{\\partial d}{\\partial c} = f = -2.0 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.9999999999988916</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-1.9999999999988916\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10 + h, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.grad = -2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1153pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1153.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1149,-150 1149,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"319,-27.5 319,-63.5 509,-63.5 509,-27.5 319,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"330.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"342,-27.5 342,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"384.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"427,-27.5 427,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"468\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"575\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"575\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M509.33,-61.52C519.71,-63.28 529.74,-64.98 538.68,-66.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"538.19,-69.97 548.63,-68.19 539.36,-63.07 538.19,-69.97\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.26,-45.5C288.54,-45.5 298.26,-45.5 308.6,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"308.69,-49 318.69,-45.5 308.69,-42 308.69,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-81.5 953,-117.5 1145,-117.5 1145,-81.5 953,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"965.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"978,-81.5 978,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1020.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1063,-81.5 1063,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1104\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"890\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"890\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M917.28,-99.5C924.78,-99.5 933.44,-99.5 942.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"942.87,-103 952.87,-99.5 942.87,-96 942.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638,-54.5 638,-90.5 827,-90.5 827,-54.5 638,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"661,-54.5 661,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"701\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"741,-54.5 741,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"784\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M827.1,-88.75C836.4,-90.37 845.38,-91.93 853.49,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"853.14,-96.83 863.59,-95.09 854.34,-89.93 853.14,-96.83\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M602.03,-72.5C609.66,-72.5 618.52,-72.5 627.94,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"627.94,-76 637.94,-72.5 627.94,-69 627.94,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-109.5 638.5,-145.5 826.5,-145.5 826.5,-109.5 638.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"659.5,-109.5 659.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"702\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"744.5,-109.5 744.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"785.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M826.65,-110.73C836.2,-109.01 845.43,-107.35 853.72,-105.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"854.47,-109.27 863.69,-104.06 853.23,-102.39 854.47,-109.27\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 512,-118.5 512,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"469\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M512.07,-83.41C521.42,-81.76 530.43,-80.17 538.55,-78.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"539.41,-82.15 548.65,-76.96 538.2,-75.25 539.41,-82.15\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derivative of `L` with respect to `e`\n",
    "\n",
    "$$ \\frac{\\partial d}{\\partial e} = \\frac{\\partial (c + e)}{\\partial e} = 1.0 $$\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial e} = \\frac{\\partial L}{\\partial d} \\cdot \\frac{\\partial d}{\\partial e} = f = -2.0 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.000000000000668</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-2.000000000000668\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    e.data += h\n",
    "    \n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "e.grad = -2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1153pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1153.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1149,-150 1149,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 511,-63.5 511,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"468\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"575\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"575\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M511.16,-61.83C520.82,-63.47 530.13,-65.05 538.49,-66.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"538.09,-69.95 548.54,-68.18 539.26,-63.05 538.09,-69.95\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.26,-45.5C288.05,-45.5 297.1,-45.5 306.75,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.99,-49 316.99,-45.5 306.99,-42 306.99,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-81.5 953,-117.5 1145,-117.5 1145,-81.5 953,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"965.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"978,-81.5 978,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1020.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1063,-81.5 1063,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1104\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"890\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"890\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M917.28,-99.5C924.78,-99.5 933.44,-99.5 942.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"942.87,-103 952.87,-99.5 942.87,-96 942.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638,-54.5 638,-90.5 827,-90.5 827,-54.5 638,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"661,-54.5 661,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"701\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"741,-54.5 741,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"784\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M827.1,-88.75C836.4,-90.37 845.38,-91.93 853.49,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"853.14,-96.83 863.59,-95.09 854.34,-89.93 853.14,-96.83\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M602.03,-72.5C609.66,-72.5 618.52,-72.5 627.94,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"627.94,-76 637.94,-72.5 627.94,-69 627.94,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-109.5 638.5,-145.5 826.5,-145.5 826.5,-109.5 638.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"659.5,-109.5 659.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"702\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"744.5,-109.5 744.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"785.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M826.65,-110.73C836.2,-109.01 845.43,-107.35 853.72,-105.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"854.47,-109.27 863.69,-104.06 853.23,-102.39 854.47,-109.27\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 512,-118.5 512,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"469\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M512.07,-83.41C521.42,-81.76 530.43,-80.17 538.55,-78.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"539.41,-82.15 548.65,-76.96 538.2,-75.25 539.41,-82.15\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derivative of `L` with respect to `a`\n",
    "\n",
    "$$ \\frac{\\partial e}{\\partial a} = \\frac{\\partial ({a}\\cdot{b})}{\\partial a} = b $$\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial a} = \\frac{\\partial L}{\\partial e} \\cdot \\frac{\\partial e}{\\partial a} = -2b = 6 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.000000000000227</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m6.000000000000227\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0 + h, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    \n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.grad = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1153pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1153.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1149,-150 1149,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-27.5 317,-63.5 511,-63.5 511,-27.5 317,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"340,-27.5 340,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"425,-27.5 425,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"468\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"575\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"575\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M511.16,-61.83C520.82,-63.47 530.13,-65.05 538.49,-66.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"538.09,-69.95 548.54,-68.18 539.26,-63.05 538.09,-69.95\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"253\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"253\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.26,-45.5C288.05,-45.5 297.1,-45.5 306.75,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.99,-49 316.99,-45.5 306.99,-42 306.99,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-81.5 953,-117.5 1145,-117.5 1145,-81.5 953,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"965.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"978,-81.5 978,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1020.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1063,-81.5 1063,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1104\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"890\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"890\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M917.28,-99.5C924.78,-99.5 933.44,-99.5 942.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"942.87,-103 952.87,-99.5 942.87,-96 942.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638,-54.5 638,-90.5 827,-90.5 827,-54.5 638,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"661,-54.5 661,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"701\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"741,-54.5 741,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"784\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M827.1,-88.75C836.4,-90.37 845.38,-91.93 853.49,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"853.14,-96.83 863.59,-95.09 854.34,-89.93 853.14,-96.83\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M602.03,-72.5C609.66,-72.5 618.52,-72.5 627.94,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"627.94,-76 637.94,-72.5 627.94,-69 627.94,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 190,-91.5 190,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.34,-56.57C199.62,-54.9 208.58,-53.29 216.66,-51.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.49,-55.25 226.72,-50.04 216.26,-48.36 217.49,-55.25\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 187.5,-36.5 187.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 6.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.65,-34.36C197.94,-36.14 207.91,-37.87 216.81,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.26,-42.87 226.71,-41.12 217.45,-35.97 216.26,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-109.5 638.5,-145.5 826.5,-145.5 826.5,-109.5 638.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"659.5,-109.5 659.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"702\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"744.5,-109.5 744.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"785.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M826.65,-110.73C836.2,-109.01 845.43,-107.35 853.72,-105.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"854.47,-109.27 863.69,-104.06 853.23,-102.39 854.47,-109.27\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"316,-82.5 316,-118.5 512,-118.5 512,-82.5 316,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-82.5 339,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"382.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"426,-82.5 426,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"469\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M512.07,-83.41C521.42,-81.76 530.43,-80.17 538.55,-78.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"539.41,-82.15 548.65,-76.96 538.2,-75.25 539.41,-82.15\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Derivative of `L` with respect to `b`\n",
    "\n",
    "$$ \\frac{\\partial e}{\\partial b} = \\frac{\\partial ({a}\\cdot{b})}{\\partial b} = a $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial L}{\\partial b} = \\frac{\\partial L}{\\partial e} \\cdot \\frac{\\partial e}{\\partial b}= -2a = -4 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.9999999999995595</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-3.9999999999995595\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def lol():\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data \n",
    "\n",
    "    a = Value(2.0, label = 'a')\n",
    "    b = Value(-3.0 + h, label='b')\n",
    "    c = Value(10, label = 'c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    \n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1) / h)\n",
    "\n",
    "lol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b.grad = -4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1157pt\" height=\"154pt\"\n",
       " viewBox=\"0.00 0.00 1157.00 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-150 1153,-150 1153,4 -4,4\"/>\n",
       "<!-- 140404042343440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404042343440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"321,-27.5 321,-63.5 515,-63.5 515,-27.5 321,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"332.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"344,-27.5 344,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"429,-27.5 429,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"472\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404042343568+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"579\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404042343440&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404042343440&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M515.16,-61.83C524.82,-63.47 534.13,-65.05 542.49,-66.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"542.09,-69.95 552.54,-68.18 543.26,-63.05 542.09,-69.95\"/>\n",
       "</g>\n",
       "<!-- 140404042343440* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404042343440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"257\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343440*&#45;&gt;140404042343440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404042343440*&#45;&gt;140404042343440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M284.26,-45.5C292.05,-45.5 301.1,-45.5 310.75,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"310.99,-49 320.99,-45.5 310.99,-42 310.99,-49\"/>\n",
       "</g>\n",
       "<!-- 140404042343504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404042343504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"957,-81.5 957,-117.5 1149,-117.5 1149,-81.5 957,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"969.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">L</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"982,-81.5 982,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1024.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;8.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1067,-81.5 1067,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1108\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343504* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404042343504*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"894\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"894\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404042343504*&#45;&gt;140404042343504 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404042343504*&#45;&gt;140404042343504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M921.28,-99.5C928.78,-99.5 937.44,-99.5 946.67,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"946.87,-103 956.87,-99.5 946.87,-96 946.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404042343568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404042343568</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"642,-54.5 642,-90.5 831,-90.5 831,-54.5 642,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"653.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"665,-54.5 665,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"705\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"745,-54.5 745,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"788\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343568&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404042343568&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M831.1,-88.75C840.4,-90.37 849.38,-91.93 857.49,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"857.14,-96.83 867.59,-95.09 858.34,-89.93 857.14,-96.83\"/>\n",
       "</g>\n",
       "<!-- 140404042343568+&#45;&gt;140404042343568 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404042343568+&#45;&gt;140404042343568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M606.03,-72.5C613.66,-72.5 622.52,-72.5 631.94,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"631.94,-76 641.94,-72.5 631.94,-69 631.94,-76\"/>\n",
       "</g>\n",
       "<!-- 140404042343120 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404042343120</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 194,-91.5 194,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"151\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343120&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404042343120&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M194.01,-56.49C203.35,-54.83 212.37,-53.23 220.49,-51.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"221.37,-55.19 230.6,-50 220.15,-48.3 221.37,-55.19\"/>\n",
       "</g>\n",
       "<!-- 140404042343184 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404042343184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4.5,-0.5 4.5,-36.5 189.5,-36.5 189.5,-0.5 4.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"27.5,-0.5 27.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"107.5,-0.5 107.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"148.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 6.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343184&#45;&gt;140404042343440* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404042343184&#45;&gt;140404042343440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M189.91,-34.21C200.76,-36.06 211.28,-37.86 220.61,-39.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"220.16,-42.93 230.6,-41.16 221.34,-36.03 220.16,-42.93\"/>\n",
       "</g>\n",
       "<!-- 140404042343248 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404042343248</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"642.5,-109.5 642.5,-145.5 830.5,-145.5 830.5,-109.5 642.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"653\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"663.5,-109.5 663.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"706\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"748.5,-109.5 748.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"789.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.0000</text>\n",
       "</g>\n",
       "<!-- 140404042343248&#45;&gt;140404042343504* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042343248&#45;&gt;140404042343504*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M830.65,-110.73C840.2,-109.01 849.43,-107.35 857.72,-105.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"858.47,-109.27 867.69,-104.06 857.23,-102.39 858.47,-109.27\"/>\n",
       "</g>\n",
       "<!-- 140404036319184 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404036319184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"320,-82.5 320,-118.5 516,-118.5 516,-82.5 320,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"331.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">c</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"343,-82.5 343,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"430,-82.5 430,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"473\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 140404036319184&#45;&gt;140404042343568+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404036319184&#45;&gt;140404042343568+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M516.07,-83.41C525.42,-81.76 534.43,-80.17 542.55,-78.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"543.41,-82.15 552.65,-76.96 542.2,-75.25 543.41,-82.15\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(L, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.data += 0.01 * a.grad\n",
    "b.data += 0.01 * b.grad\n",
    "c.data += 0.01 * c.grad\n",
    "f.data += 0.01 * f.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-7.286496</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-7.286496\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "e = a * b\n",
    "d = e + c\n",
    "L = d * f\n",
    "print(L.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGdCAYAAAAfTAk2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHXElEQVR4nO3de1yUdd4//tfMMAwHYRSQARQBjRREUyERTM1VUMvOnu6K1lZt/dnJqF/FWpu623J32KQsK3fdJQ+pd7dr1h0Zo+WhQPMAmqakpoLIWWBQZI7X9w9kcuQgo8xczDWv5+MxD5hrPnPxvt4N+Oo6fC6ZIAgCiIiIiCRELnYBRERERF2NAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkx0PsAsRgsVhw/vx5+Pn5QSaTiV0OERERdYIgCGhoaEBYWBjk8o730bhlwDl//jzCw8PFLoOIiIhuQElJCfr27dvhGLcMOH5+fgCaG+Tv7y9yNd2D0WhEbm4uUlNToVQqxS5H8thv52K/nYv9di536rdOp0N4eLj13/GOuGXAaTks5e/vz4BzhdFohI+PD/z9/SX/C9IdsN/OxX47F/vtXO7Y786cXsKTjImIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIcGnB27dqFe+65B2FhYZDJZPj888+v+56dO3ciPj4eXl5e6N+/Pz766KNWYzZt2oTY2FioVCrExsZi8+bNDqieiIiIXJVDA86lS5dw22234f333+/U+NOnT+Ouu+7CmDFjUFBQgD/96U945plnsGnTJuuY/Px8zJw5E2lpaTh06BDS0tIwY8YM7N2711GbQURERC7GofeimjJlCqZMmdLp8R999BH69euHrKwsAEBMTAz279+Pt99+Gw899BAAICsrCykpKcjIyAAAZGRkYOfOncjKysL69eu7fBuIiIjI9XSrm23m5+cjNTXVZtmkSZOwatUqGI1GKJVK5Ofn47nnnms1piUUtUWv10Ov11uf63Q6AM03KDMajV23AS6spQ/sh3Ow387FfjuXlPtttggwmCwwWSwwmgWYLAJMZguMFgEm85XvzQJMFgvMFgFmQWj+agHMFgvMAmA2Ny+3XHndIgDClXEt31sEwCwI1u9bvgKARRAgCL99NZnNOHFOhlPbT0Aml0MAAAEQ0Py6AFz5euUJflvW/L1w1fdXvrYsuMq1Y64eJ7QxLtDXEwvu7H9T/b6WPZ+pbhVwysvLodFobJZpNBqYTCZUV1cjNDS03THl5eXtrjczMxNLlixptTw3Nxc+Pj5dU7xEaLVasUtwK+y3c7HfziVWvy0CcNkENJqAy2ag0STDZRPQZAYMlisPswz6K8+v/mq0yGASAJMFMFrQ6nuLcP27WItDAZScFrsIG8FeAiIbj3fpOhsbGzs9tlsFHKD1LdBb0uHVy9sa09Gt0zMyMpCenm59rtPpEB4ejtTUVPj7+3dF2S7PaDRCq9UiJSUFSqVS7HIkj/12Lvbbubq634Ig4KLehMoGA6oa9Khs0KPqot76ffVFA+oajdA1GVF/2YSLelMXbEXnyGWAh0IOpVwGD4UMHnI5PBQyKOUyKORyKOQyKOSAQiaDQiFr/ipvfshlv32Vy9D8VX7lq0wGWcsyWfO/ezLgquUArrwmWASUnS9F3759IJcrIJMBV16GDDLrc1xZR8s/l83fy6763nbbbP7dtS5reS67Zixaje3l64m7kiNussO2Wo7AdEa3CjghISGt9sRUVlbCw8MDgYGBHY65dq/O1VQqFVQqVavlSqWSf+yuwZ44F/vtXOy3c9nTb4PJgpLaRpypvoTT1ZdwpuYSzlQ3oqS2ERW6JjQZLXb/fB9PBdTeSqi9lfD3UqKHlwe8PRXwUSrgq/rtex+VB3w8FfDxVEDloYCXUg5PDzlUHgqoPORXHoory+RQesihVMiglMshl4u/R8doNCInpwR33TVE8p9ve7avWwWcpKQkfPnllzbLcnNzkZCQYN2opKQkaLVam/NwcnNzkZyc7NRaiYjIfiazBScqL6KwpA7Hy3Q4XdMcas7VNlrPMWmPn5cHgv1UCPbzQrC/yvp9bz8VevoorWFG7a2En5cSnh6c6s2dOTTgXLx4ESdPnrQ+P336NAoLCxEQEIB+/fohIyMDpaWlWL16NQBg/vz5eP/995Geno558+YhPz8fq1atsrk66tlnn8XYsWPxxhtv4L777sOWLVuwbds2fP/9947cFCIispMgCDhfdxmHSupQWFKHgpI6HCmtR6PB3OZ4H08FIgN9ERXki8ggH0QG+iIi0Bch/s0hxttT4eQtIFfm0ICzf/9+jB8/3vq85TyY3//+98jOzkZZWRmKi4utr0dFRSEnJwfPPfccPvjgA4SFheG9996zXiIOAMnJydiwYQNeeeUVvPrqqxgwYAA2btyIxMRER24KERF1wtmaS9j603n833E5Xj+yC5UN+lZjeqg8MLSvGkP6qNG/t6811PT2U3V4PiWRPRwacO688842LzVrkZ2d3WrZuHHjcPDgwQ7XO23aNEybNu1myyMiopskCAKOntch92g5cn+uwPHyhiuvyAHooZDLMFDjh2H9emJYeE8MD++JAb17dItzV0jautU5OERE1P2ZzBbsP1uLb46WI/doBUrrLltfU8hlGBnZC71NVZiVMgrDIwJ5aIlEwYBDRESdcqxMh0/yzuCbo+WobfxtwjUvpRxjo3tj0uAQ/G5QMHp4ypCTk4PbI3tBqWS4IXEw4BARUbsEQUD+qRp8tOtX7Pqlyrq8p48SEwZpMGmwBmOie9vspZHiDMbkehhwiIioFZPZgq+PlOPjXadwpLR5cjW5DJgyJBSPjOyHkVEB8FDwMmzqvhhwiIjIqtFgwmf7z+Gf3/+KkgvN59Z4KeWYkRCOuXf0R79A3t6GXAMDDhER4aLehJW7fsXq/DOou3J+TYCvJx5LisBjSZEI8PUUuUIi+zDgEBG5ud0nqvDypp+sV0P1C/DBvDFRmBYfziugyGUx4BARuSldkxGZOcew/scSAEB4gDdenhyDyXEhUHCeGnJxDDhERG5oR1ElMv7zE8rqmwAAs5Mj8eLkgfDx5D8LJA38JBMRuZH6y0a8/tXP+J/95wAAEYE+ePOhoUjsHyhyZURdiwGHiMhNfHe8ea9Nua4JMhnweHIU/v9JA3meDUkSAw4RkcTVXzZi6Zc/Y9PB5r02UUG+eGvaUCREBohcGZHjMOAQEUlYpa4Jj67ai18qLkImA+beEYX0FO61IeljwCEikqiSC414dNVenK1phMZfhRWPjEB8BPfakHtgwCEikqCTlRfx6D/3olzXhH4BPlg3NxHhAZyFmNwHAw4RkcQcKa3HY//6ERcuGRAd3ANr5yZC4+8ldllETsWAQ0QkIfvPXMDj2fvQ0GTCkD5qfPKHkbzNArklBhwiIonYfaIKT6w+gMtGM0ZGBuCfsxPg76UUuywiUTDgEBFJwDdHy/H0pwUwmC0Yd2tvfPRoPK+UIrfGgENE5OI2F5zDC58dhtkiYEpcCN6dNRyeHnKxyyISFQMOEZELW/9jMTL+8xMAYFp8X/z3g0PgoWC4IWLAISJyUXt/rcErnx8B0HyzzD9PjYWcdwEnAsCAQ0Tkkqoa9Hh6fQHMFgEPDO+D1+6JhUzGcEPUgvsxiYhcjNkiYOHGAlQ26BEd3AOvPxDHcEN0DQYcIiIX8+72E/jhZA28lQp8+OgI+HhyZzzRtRhwiIhcyK5fqrD82xMAgMwHh+CWYD+RKyLqnhhwiIhcRFn9ZSzcWAhBAB5O7If7h/cRuySibosBh4jIBRjNFjz9aQEuXDJgcJg//jw1VuySiLo1BhwiIhfw1jdF2H+2Fn4qD6x4ZAS8lJylmKgjDDhERN1c7tFyrNz1KwDgrelDERHoK3JFRN0fAw4RUTdWXNOI5z87BACYc0cUJseFilwRkWtwSsBZsWIFoqKi4OXlhfj4eOzevbvdsbNnz4ZMJmv1GDx4sHVMdnZ2m2OampqcsTlERE7RZDTjyU8PoqHJhOH9euKlyYPELonIZTg84GzcuBELFy7EokWLUFBQgDFjxmDKlCkoLi5uc/y7776LsrIy66OkpAQBAQGYPn26zTh/f3+bcWVlZfDy8nL05hAROc3rXx3DT6X16OWjxAcPj+ANNIns4PDflnfeeQdz5szB3LlzERMTg6ysLISHh+PDDz9sc7xarUZISIj1sX//ftTW1uLxxx+3GSeTyWzGhYSEOHpTiIicZt+ZC1iz5ywA4J2ZwxDW01vkiohci0OnvzQYDDhw4ABefvllm+WpqanIy8vr1DpWrVqFiRMnIiIiwmb5xYsXERERAbPZjGHDhuEvf/kLhg8f3uY69Ho99Hq99blOpwMAGI1GGI1GezZJslr6wH44B/vtXK7Wb4tFwF++PAoAmJnQB3f07+UytQOu129X5079tmcbHRpwqqurYTabodFobJZrNBqUl5df9/1lZWX4+uuv8emnn9osHzRoELKzszFkyBDodDq8++67GD16NA4dOoTo6OhW68nMzMSSJUtaLc/NzYWPj4+dWyVtWq1W7BLcCvvtXK7S7/1VMhwuVUAlFxAnnEVOzlmxS7ohrtJvqXCHfjc2NnZ6rFNuYHLtTeAEQejUjeGys7PRs2dP3H///TbLR40ahVGjRlmfjx49GiNGjMDy5cvx3nvvtVpPRkYG0tPTrc91Oh3Cw8ORmpoKf39/O7dGmoxGI7RaLVJSUqBUKsUuR/LYb+dypX43Gc3473d/ANCEJ38XjVnj+otdkt1cqd9S4E79bjkC0xkODThBQUFQKBSt9tZUVla22qtzLUEQ8K9//QtpaWnw9PTscKxcLsftt9+OEydOtPm6SqWCSqVqtVypVEr+w2Av9sS52G/ncoV+r/z+LMrqmxCm9sIT426B0oUn9HOFfkuJO/Tbnu1z6EnGnp6eiI+Pb7XbTKvVIjk5ucP37ty5EydPnsScOXOu+3MEQUBhYSFCQzk/BBG5rsqGJqz47iQA4KUpgzhbMdFNcPghqvT0dKSlpSEhIQFJSUlYuXIliouLMX/+fADNh49KS0uxevVqm/etWrUKiYmJiIuLa7XOJUuWYNSoUYiOjoZOp8N7772HwsJCfPDBB47eHCIih1mm/QWXDGbc1leNe4aGiV0OkUtzeMCZOXMmampqsHTpUpSVlSEuLg45OTnWq6LKyspazYlTX1+PTZs24d13321znXV1dXjiiSdQXl4OtVqN4cOHY9euXRg5cqSjN4eIyCGOl+uwcV8JAOCVqbGQy69/niIRtc8pJxkvWLAACxYsaPO17OzsVsvUanWHZ0ovW7YMy5Yt66ryiIhEJQgCXv/qGCwCcNeQENweGSB2SUQuj9NiEhGJbMcvVdh9ohqeCjlvx0DURRhwiIhEZDJb8PpXxwAAv0+O4J3CiboIAw4RkYg27CvBycqL6OWjxFO/az1RKRHdGAYcIiKR6JqMWKb9BQCwcOKtUHtLew4TImdiwCEiEsmK706h5pIB/Xv74uHEfmKXQyQpDDhERCIoudCIf/1wGgDwpykxUCr455ioK/E3iohIBG9+UwSDyYLkAYGYEBMsdjlEksOAQ0TkZD+dq8eXh85DJgMW3R3TqZsPE5F9GHCIiJys5dDUvbeFYXCYWuRqiKSJAYeIyIkqG5rwf4fPAwDm3BElcjVE0sWAQ0TkROv2FMNoFhAf0QtD+/YUuxwiyWLAISJyEr3JjHV7m28uPDs5UtxiiCSOAYeIyEm+OlyG6ot6hPh7YXJciNjlEEkaAw4RkRMIgoB//3AGAJCWFMF5b4gcjL9hREROcLC4Fj+V1sPTQ45Zt4eLXQ6R5DHgEBE5Qcvem/uHhSGwh0rcYojcAAMOEZGDldVfxtdHygEAs5N5aTiRMzDgEBE52No9Z2G2CEiMCkBsmL/Y5RC5BQYcIiIHajKa8emVS8MfHx0pbjFEboQBh4jIgb4oPI/aRiP69PTGxBiN2OUQuQ0GHCIiBxEEAf/OOwMAeCwpAh68NJzIafjbRkTkIHtPX8CxMh28lHLM5KXhRE7FgENE5CDZVy4Nf3BEX/T08RS3GCI3w4BDROQAJRcakftzy6XhkeIWQ+SGGHCIiBxg7Z6zsAjA6FsCcavGT+xyiNwOAw4RURdrNJiw/scrl4ZzYj8iUTDgEBF1sc0FpdA1mdAvwAfjBwWLXQ6RW2LAISLqQoIgWE8u/n1yJBRymbgFEbkpBhwioi70w8kanKi8CB9PBaYn9BW7HCK3xYBDRNSFPjtQAgB4cEQf+HspRa6GyH0x4BARdZHLBjO0P1cAaJ77hojE45SAs2LFCkRFRcHLywvx8fHYvXt3u2N37NgBmUzW6nH8+HGbcZs2bUJsbCxUKhViY2OxefNmR28GEVGHth2rQKPBjPAAbwwP7yl2OURuzeEBZ+PGjVi4cCEWLVqEgoICjBkzBlOmTEFxcXGH7ysqKkJZWZn1ER0dbX0tPz8fM2fORFpaGg4dOoS0tDTMmDEDe/fudfTmEBG1a0vheQDAPUPDIJPx5GIiMTk84LzzzjuYM2cO5s6di5iYGGRlZSE8PBwffvhhh+8LDg5GSEiI9aFQKKyvZWVlISUlBRkZGRg0aBAyMjIwYcIEZGVlOXhriIjaVt9oxM5fKgEA9w3rI3I1ROThyJUbDAYcOHAAL7/8ss3y1NRU5OXldfje4cOHo6mpCbGxsXjllVcwfvx462v5+fl47rnnbMZPmjSp3YCj1+uh1+utz3U6HQDAaDTCaDTas0mS1dIH9sM52G/ncka/vzp8DkazgFuDe6B/oJdb/7fl59u53Knf9myjQwNOdXU1zGYzNBqNzXKNRoPy8vI23xMaGoqVK1ciPj4eer0ea9aswYQJE7Bjxw6MHTsWAFBeXm7XOjMzM7FkyZJWy3Nzc+Hj43MjmyZZWq1W7BLcCvvtXI7sd/bPcgByRKvqkZOT47Cf40r4+XYud+h3Y2Njp8c6NOC0uPZYtCAI7R6fHjhwIAYOHGh9npSUhJKSErz99tvWgGPvOjMyMpCenm59rtPpEB4ejtTUVPj7+9u9PVJkNBqh1WqRkpICpZKXtjoa++1cju53ZYMeJ/bsBACkTxuHfgHu/T9O/Hw7lzv1u+UITGc4NOAEBQVBoVC02rNSWVnZag9MR0aNGoW1a9dan4eEhNi1TpVKBZVK1Wq5UqmU/IfBXuyJc7HfzuWofn/z8zkIAjC8X08M0Ki7fP2uip9v53KHftuzfQ49ydjT0xPx8fGtdptptVokJyd3ej0FBQUIDQ21Pk9KSmq1ztzcXLvWSUTUVb441Hz11L23hYlcCRG1cPghqvT0dKSlpSEhIQFJSUlYuXIliouLMX/+fADNh49KS0uxevVqAM1XSEVGRmLw4MEwGAxYu3YtNm3ahE2bNlnX+eyzz2Ls2LF44403cN9992HLli3Ytm0bvv/+e0dvDhGRjeKaRhSW1EEuA+4eGnr9NxCRUzg84MycORM1NTVYunQpysrKEBcXh5ycHERERAAAysrKbObEMRgMeOGFF1BaWgpvb28MHjwYX331Fe666y7rmOTkZGzYsAGvvPIKXn31VQwYMAAbN25EYmKiozeHiMjGl4eb994kDQhEsJ+XyNUQUQunnGS8YMECLFiwoM3XsrOzbZ6/+OKLePHFF6+7zmnTpmHatGldUR4R0Q3bUlgKALjvNs59Q9Sd8F5UREQ36Hi5Dr9UXISnQo5JcSFil0NEV2HAISK6QV9cuTXDuIG9ofaW9tUrRK6GAYeI6AYIgmA9/+a+Ybx6iqi7YcAhIroBBSV1KLlwGb6eCkwY1Pl5vYjIORhwiIhuQMvhqZRYDbw9FdcZTUTOxoBDRGQnk9mC/ztcBgC4l4eniLolBhwiIjvt+fUCqi/q0ctHiTHRvcUuh4jawIBDRGSnLw41z30zZUgolAr+GSXqjvibSURkB73JjK+PNN/sl/eeIuq+GHCIiOywo6gKDU0mhPh7YWRkgNjlEFE7GHCIiOzQcufwe24LhVwuE7kaImoPAw4RUSdd0puw/VgFAOBe3nuKqFtjwCEi6iTtzxVoMloQFeSLuD7+YpdDRB1gwCEi6qSWuW/uuS0MMhkPTxF1Zww4RESd0GQ04/uTVQCAKbxzOFG3x4BDRNQJP5ysRpPRgj49vTEoxE/scojoOhhwiIg6YduxSgDA7wYF8/AUkQtgwCEiug5BEPDt8earpybEBItcDRF1BgMOEdF1HD2vQ4VODx9PBUb1DxS7HCLqBAYcIqLr2HZl7psx0UHwUipEroaIOoMBh4joOrZfOf9mQoxG5EqIqLMYcIiIOlCha8JPpfWQyYDxA3n+DZGrYMAhIupAy96b2/r2RG8/lcjVEFFnMeAQEXWg5eqpibx6isilMOAQEbWjefbiagA8/4bI1TDgEBG1o2X24jC1F2cvJnIxDDhERO3YdtXVU5y9mMi1MOAQEbWBsxcTuTYGHCKiNhwp5ezFRK6MAYeIqA3bj3P2YiJXxoBDRNQG6+zFg3j1FJErckrAWbFiBaKiouDl5YX4+Hjs3r273bH/+c9/kJKSgt69e8Pf3x9JSUn45ptvbMZkZ2dDJpO1ejQ1NTl6U4jIDdjMXjyI598QuSKHB5yNGzdi4cKFWLRoEQoKCjBmzBhMmTIFxcXFbY7ftWsXUlJSkJOTgwMHDmD8+PG45557UFBQYDPO398fZWVlNg8vLy9Hbw4RuQHOXkzk+jwc/QPeeecdzJkzB3PnzgUAZGVl4ZtvvsGHH36IzMzMVuOzsrJsnv/tb3/Dli1b8OWXX2L48OHW5TKZDCEhIQ6tnYjc0/ZjnL2YyNU5NOAYDAYcOHAAL7/8ss3y1NRU5OXldWodFosFDQ0NCAgIsFl+8eJFREREwGw2Y9iwYfjLX/5iE4Cuptfrodfrrc91Oh0AwGg0wmg02rNJktXSB/bDOdhv57Kn35cNv81ePPaWQP43ugH8fDuXO/Xbnm10aMCprq6G2WyGRmN7kp5Go0F5eXmn1vH3v/8dly5dwowZM6zLBg0ahOzsbAwZMgQ6nQ7vvvsuRo8ejUOHDiE6OrrVOjIzM7FkyZJWy3Nzc+Hj42PnVkmbVqsVuwS3wn47V2f6faRWBr1JgZ6eAn49uBunOb/fDePn27ncod+NjY2dHuvwQ1QAWs0AKghCp2YFXb9+PRYvXowtW7YgOPi3XcWjRo3CqFGjrM9Hjx6NESNGYPny5XjvvfdarScjIwPp6enW5zqdDuHh4UhNTYW/v/+NbJLkGI1GaLVapKSkQKlUil2O5LHfzmVPv/O2/AzgHO4e1g933x3jnAIlhp9v53KnfrccgekMhwacoKAgKBSKVntrKisrW+3VudbGjRsxZ84cfPbZZ5g4cWKHY+VyOW6//XacOHGizddVKhVUqtYnCiqVSsl/GOzFnjgX++1c1+u3IAjY8UsVACBlcAj/29wkfr6dyx36bc/2OfQqKk9PT8THx7fababVapGcnNzu+9avX4/Zs2fj008/xd13333dnyMIAgoLCxEaGnrTNROR++LsxUTS4fBDVOnp6UhLS0NCQgKSkpKwcuVKFBcXY/78+QCaDx+VlpZi9erVAJrDzWOPPYZ3330Xo0aNsu798fb2hlqtBgAsWbIEo0aNQnR0NHQ6Hd577z0UFhbigw8+cPTmEJGEtcxefMctnL2YyNU5PODMnDkTNTU1WLp0KcrKyhAXF4ecnBxEREQAAMrKymzmxPn4449hMpnw5JNP4sknn7Qu//3vf4/s7GwAQF1dHZ544gmUl5dDrVZj+PDh2LVrF0aOHOnozSEiCWuZ/2ZiDGcvJnJ1TjnJeMGCBViwYEGbr7WElhY7duy47vqWLVuGZcuWdUFlRETNOHsxkbTwXlRERODsxURSw4BDRATOXkwkNQw4ROT2moxm/HCqefZiHp4ikgYGHCJye/vP1KLJaEGwnwqxoZz8k0gKGHCIyO3tOtE8ud+Y6N6dmmWdiLo/Bhwicnu7rsxePPbWIJErIaKuwoBDRG6tQteE4+UNkMma9+AQkTQw4BCRW2vZezOkjxoBvp4iV0NEXYUBh4jc2q4TzVdPjeXeGyJJYcAhIrdltgj4/kTL+TcMOERSwoBDRG7rSGk9ahuN6KHywPB+PcUuh4i6EAMOEbmtlvNvkgcEQqngn0MiKeFvNBG5rV08PEUkWQw4ROSWdE1GHCyuAwCMY8AhkhwGHCJyS3kna2C2CIgK8kV4gI/Y5RBRF2PAISK3ZD08Fc3Zi4mkiAGHiNyOIAhX3Z6Bh6eIpIgBh4jczunqSzhXexlKhQyj+geKXQ4ROQADDhG5nZa9NwkRAfBVeYhcDRE5AgMOEbkd6+0ZeHiKSLIYcIjIrehNZuSfqgEAjL2VJxgTSRUDDhG5lQNnanHZaEZQDxViQvzFLoeIHIQBh4jcyk7r7MVBkMtlIldDRI7CgENEbmXXL83n33D2YiJpY8AhIrdR1aDHsTIdZDLgjlt4/g2RlDHgEJHb+P5k88nFcWFqBPZQiVwNETkSAw4RuY3dJ1suD+feGyKpY8AhIrdgEX7bgzM2muffEEkdAw4RuYXSS0BtoxE9VB4YEdFL7HKIyMEYcIjILRyra74kPGlAIJQK/ukjkjr+lhORWzhe1/znjrdnIHIPTgk4K1asQFRUFLy8vBAfH4/du3d3OH7nzp2Ij4+Hl5cX+vfvj48++qjVmE2bNiE2NhYqlQqxsbHYvHmzo8onIhfX0GTC6YvN34/j+TdEbsHhAWfjxo1YuHAhFi1ahIKCAowZMwZTpkxBcXFxm+NPnz6Nu+66C2PGjEFBQQH+9Kc/4ZlnnsGmTZusY/Lz8zFz5kykpaXh0KFDSEtLw4wZM7B3715Hbw4RuaA9v16ARZAhIsAH/QJ9xC6HiJzA4QHnnXfewZw5czB37lzExMQgKysL4eHh+PDDD9sc/9FHH6Ffv37IyspCTEwM5s6diz/84Q94++23rWOysrKQkpKCjIwMDBo0CBkZGZgwYQKysrIcvTlE5IJaLg8fEx0ociVE5Cwejly5wWDAgQMH8PLLL9ssT01NRV5eXpvvyc/PR2pqqs2ySZMmYdWqVTAajVAqlcjPz8dzzz3Xakx7AUev10Ov11uf63Q6AIDRaITRaLR3sySppQ/sh3Ow384jCAJ2n2gOOEmRPdlzJ+Dn27ncqd/2bKNDA051dTXMZjM0Go3Nco1Gg/Ly8jbfU15e3uZ4k8mE6upqhIaGtjumvXVmZmZiyZIlrZbn5ubCx4e7q6+m1WrFLsGtsN+OV3kZOFfnAYVMwKXTBcgpLhC7JLfBz7dzuUO/GxsbOz3WoQGnhUxme8deQRBaLbve+GuX27POjIwMpKenW5/rdDqEh4cjNTUV/v7+ndsIiTMajdBqtUhJSYFSqRS7HMljv51nzZ5ioPA4ovwETJ3MfjsDP9/O5U79bjkC0xkODThBQUFQKBSt9qxUVla22gPTIiQkpM3xHh4eCAwM7HBMe+tUqVRQqVrfd0apVEr+w2Av9sS52G/H++HUBQDAoJ4C++1k7LdzuUO/7dk+h55k7Onpifj4+Fa7zbRaLZKTk9t8T1JSUqvxubm5SEhIsG5Ye2PaWycRuSeDyYL8X5tvzxDTUxC5GiJyJocfokpPT0daWhoSEhKQlJSElStXori4GPPnzwfQfPiotLQUq1evBgDMnz8f77//PtLT0zFv3jzk5+dj1apVWL9+vXWdzz77LMaOHYs33ngD9913H7Zs2YJt27bh+++/d/TmEJEL2X/2AhoNZgT6eiLMxyR2OUTkRA4PODNnzkRNTQ2WLl2KsrIyxMXFIScnBxEREQCAsrIymzlxoqKikJOTg+eeew4ffPABwsLC8N577+Ghhx6yjklOTsaGDRvwyiuv4NVXX8WAAQOwceNGJCYmOnpziMiF7Pql+eqpO24JhFzW+ZMTicj1OeUk4wULFmDBggVtvpadnd1q2bhx43Dw4MEO1zlt2jRMmzatK8ojIona9UsVAGDMLYHA+RKRqyEiZ+K9qIhIkqoa9Pi5rPmKiztu4QR/RO6GAYeIJGn3iea9N4PD/BHYo/VVlEQkbQw4RCRJLYenePdwIvfEgENEkmOx/HZ7hnEMOERuiQGHiCTn5zIdai4Z4OupwIh+vcQuh4hEwIBDRJKz88rhqaQBQfD04J85InfE33wikpyW82/G3RokciVEJBYGHCKSlIt6Ew6crQXAE4yJ3BkDDhFJSv6pGpgsAiICfRAR6Ct2OUQkEgYcIpIU6+Xh0dx7Q+TOGHCISFJ2neD8N0TEgENEEnK25hLO1jTCQy5D0gDenoHInTHgEJFktByeio/ohR4qp9xLmIi6KQYcIpKMnb80z17Mw1NExIBDRJJgMFmQf4q3ZyCiZgw4RCQJB4trcclgRqCvJ2JD/cUuh4hExoBDRJLQcv7NmOggyOUykashIrEx4BCRJPDycCK6GgMOEbm8qgY9jpTqAABjOMEfEYEBh4gk4PuTzXtvYkP90dtPJXI1RNQdMOAQkcvbxcvDiegaDDhE5NIsFgG7r5x/w8vDiagFAw4RubSfy3SovmiAr6cC8RG9xC6HiLoJBhwicmktV08lDQiEpwf/pBFRM/41ICKX1jL/Dc+/IaKrMeAQkcu6pDfhwNlaAMBYXh5ORFdhwCEil5V3qgZGs4B+AT6IDPIVuxwi6kYYcIjIZX17vAIAMH4g994QkS0GHCJySRaLgO3HKgEAE2I0IldDRN0NAw4RuaQj5+tR2aCHr6cCif0DxC6HiLoZBhwicknbruy9GXtrb6g8FCJXQ0TdjUMDTm1tLdLS0qBWq6FWq5GWloa6urp2xxuNRrz00ksYMmQIfH19ERYWhsceewznz5+3GXfnnXdCJpPZPGbNmuXITSGibmb7sebzb343KFjkSoioO3JowHn44YdRWFiIrVu3YuvWrSgsLERaWlq74xsbG3Hw4EG8+uqrOHjwIP7zn//gl19+wb333ttq7Lx581BWVmZ9fPzxx47cFCLqRsrqL+PoeR1kMmA8Aw4RtcHDUSs+duwYtm7dij179iAxMREA8I9//ANJSUkoKirCwIEDW71HrVZDq9XaLFu+fDlGjhyJ4uJi9OvXz7rcx8cHISEhjiqfiLqxb483H54aHt4TQT1493Aias1hASc/Px9qtdoabgBg1KhRUKvVyMvLazPgtKW+vh4ymQw9e/a0Wb5u3TqsXbsWGo0GU6ZMwWuvvQY/P78216HX66HX663PdTodgOZDYkaj0c4tk6aWPrAfzsF+3xzt0XIAwPhbgzrVQ/bbudhv53KnftuzjQ4LOOXl5QgObr3rODg4GOXl5Z1aR1NTE15++WU8/PDD8Pf3ty5/5JFHEBUVhZCQEBw5cgQZGRk4dOhQq70/LTIzM7FkyZJWy3Nzc+Hj49PJLXIP7fWQHIP9tp/BDPxwQgFABo+q48jJOd7p97LfzsV+O5c79LuxsbHTY+0OOIsXL24zLFxt3759AACZTNbqNUEQ2lx+LaPRiFmzZsFisWDFihU2r82bN8/6fVxcHKKjo5GQkICDBw9ixIgRrdaVkZGB9PR063OdTofw8HCkpqbaBCd3ZjQaodVqkZKSAqVSKXY5ksd+37jtxyph/LEQfXp6Yc5DYzr994T9dh7227ncqd8tR2A6w+6A89RTT133iqXIyEgcPnwYFRUVrV6rqqqCRtPxpFxGoxEzZszA6dOn8e233143hIwYMQJKpRInTpxoM+CoVCqoVK2P0yuVSsl/GOzFnjgX+22/HSdqAAATYzTw9PS0673st3Ox387lDv22Z/vsDjhBQUEICgq67rikpCTU19fjxx9/xMiRIwEAe/fuRX19PZKTk9t9X0u4OXHiBL777jsEBgZe92cdPXoURqMRoaGhnd8QInI5FotgPcGYsxcTUUccdpl4TEwMJk+ejHnz5mHPnj3Ys2cP5s2bh6lTp9qcYDxo0CBs3rwZAGAymTBt2jTs378f69atg9lsRnl5OcrLy2EwGAAAp06dwtKlS7F//36cOXMGOTk5mD59OoYPH47Ro0c7anOIqBvg7MVE1FkOnQdn3bp1GDJkCFJTU5GamoqhQ4dizZo1NmOKiopQX18PADh37hy++OILnDt3DsOGDUNoaKj1kZeXBwDw9PTE9u3bMWnSJAwcOBDPPPMMUlNTsW3bNigUnM2USMpaZi8eE83Zi4moYw67igoAAgICsHbt2g7HCIJg/T4yMtLmeVvCw8Oxc+fOLqmPiFxLy+zFE2I4uR8RdYz3oiIil8DZi4nIHgw4ROQSOHsxEdmDAYeIXML2Y7x6iog6jwGHiLq9ywYzfjhZDYDn3xBR5zDgEFG39/3JauhNFvTp6Y2BmrbvOUdEdDUGHCLq9lqunpoYE9ypWzMQETHgEFG3ZrEI2M7Zi4nITgw4RNStHTlfjyrOXkxEdmLAIaJujbMXE9GNYMAhom6NsxcT0Y1gwCGibouzFxPRjWLAIaJuq2VyP85eTET2YsAhom7rW149RUQ3iAGHiLolzl5MRDeDAYeIuiXOXkxEN4MBh4i6pa+PlAHg7MVEdGMYcIio22kympF7tPny8HtuCxO5GiJyRQw4RNTtfHu8Ehf1JvTp6Y0R/XqJXQ4RuSAGHCLqdr4oPA8AmHpbKORyHp4iIvsx4BBRt6JrMuLboubLw+/l4SkiukEMOETUreQerYDBZMEtwT0QG+ovdjlE5KIYcIioW9lSWAqgee8Nr54iohvFgENE3Ub1RT3yTtUA4OEpIro5DDhE1G3k/FQGs0XA0L5qRAb5il0OEbkwBhwi6jZarp7i3hsiulkMOETULZyrbcT+s7WQyTi5HxHdPAYcIuoWvjzUfGuGxKgAaPy9RK6GiFwdAw4RdQtfHGo5PNVH5EqISAoYcIhIdCcrG3CsTAelQoYpcSFil0NEEsCAQ0Siazm5eGx0b/Ty9RS5GiKSAgYcIhKVIAi/HZ4axpOLiahrODTg1NbWIi0tDWq1Gmq1Gmlpaairq+vwPbNnz4ZMJrN5jBo1ymaMXq/H008/jaCgIPj6+uLee+/FuXPnHLglROQoh8/V40xNI7yUckyM0YhdDhFJhEMDzsMPP4zCwkJs3boVW7duRWFhIdLS0q77vsmTJ6OsrMz6yMnJsXl94cKF2Lx5MzZs2IDvv/8eFy9exNSpU2E2mx21KUTkIC17b1JiQ+Cr8hC5GiKSCof9NTl27Bi2bt2KPXv2IDExEQDwj3/8A0lJSSgqKsLAgQPbfa9KpUJISNsnGtbX12PVqlVYs2YNJk6cCABYu3YtwsPDsW3bNkyaNKnrN4aIHMJsEfB/hzm5HxF1PYcFnPz8fKjVamu4AYBRo0ZBrVYjLy+vw4CzY8cOBAcHo2fPnhg3bhxef/11BAcHAwAOHDgAo9GI1NRU6/iwsDDExcUhLy+vzYCj1+uh1+utz3U6HQDAaDTCaDTe9LZKQUsf2A/nYL+b7T19ARU6Pfy9PJAU1dNh/WC/nYv9di536rc92+iwgFNeXm4NJVcLDg5GeXl5u++bMmUKpk+fjoiICJw+fRqvvvoqfve73+HAgQNQqVQoLy+Hp6cnevXqZfM+jUbT7nozMzOxZMmSVstzc3Ph4+Nj55ZJm1arFbsEt+Lu/d5wSg5Ajlh/A7bnbnX4z3P3fjsb++1c7tDvxsbGTo+1O+AsXry4zbBwtX379gEAZDJZq9cEQWhzeYuZM2dav4+Li0NCQgIiIiLw1Vdf4cEHH2z3fR2tNyMjA+np6dbnOp0O4eHhSE1Nhb+/f4fb4i6MRiO0Wi1SUlKgVCrFLkfy2G/AYLLgz2/uAGDCgrtvR1L/QIf9LPbbudhv53KnfrccgekMuwPOU089hVmzZnU4JjIyEocPH0ZFRUWr16qqqqDRdP5KidDQUERERODEiRMAgJCQEBgMBtTW1trsxamsrERycnKb61CpVFCpVK2WK5VKyX8Y7MWeOJc793vXyQrUXzYh2E+F0dEaKOTt/49PV3HnfouB/XYud+i3Pdtnd8AJCgpCUFDQdcclJSWhvr4eP/74I0aOHAkA2Lt3L+rr69sNIm2pqalBSUkJQkNDAQDx8fFQKpXQarWYMWMGAKCsrAxHjhzBm2++ae/mEJFIWq6euntoqFPCDRG5F4ddJh4TE4PJkydj3rx52LNnD/bs2YN58+Zh6tSpNicYDxo0CJs3bwYAXLx4ES+88ALy8/Nx5swZ7NixA/fccw+CgoLwwAMPAADUajXmzJmD559/Htu3b0dBQQEeffRRDBkyxHpVFRF1b40GE7Q/N+/h5dVTROQIDp10Yt26dXjmmWesVzzde++9eP/9923GFBUVob6+HgCgUCjw008/YfXq1airq0NoaCjGjx+PjRs3ws/Pz/qeZcuWwcPDAzNmzMDly5cxYcIEZGdnQ6FQOHJziKiLbDtWiUaDGf0CfDAsvKfY5RCRBDk04AQEBGDt2rUdjhEEwfq9t7c3vvnmm+uu18vLC8uXL8fy5ctvukYicr7NB5tnHr/3trAOLzogIrpRvBcVETnV6epL+K6oCjIZ8FB8X7HLISKJYsAhIqf6JO8MAGD8wGBEBfmKWwwRSRYDDhE5TUOTEf97oPnw1OOjI8UthogkjQGHiJzmfw+cw0W9CbcE98Adt1x/ugkiohvFgENETmGxCNbDU7OTI3lyMRE5FAMOETnFjl8qcaamEX5eHnhwRB+xyyEiiWPAISKn+PcPZwAAs24Ph4+nQ2eoICJiwCEixztZ2YDdJ6ohlwGPJUWKXQ4RuQEGHCJyuOwr595MjNEgPMBH3GKIyC0w4BCRQ9U3GrHpQCkAYDYvDSciJ2HAISKH+p/9JbhsNGNQiB+S+geKXQ4RuQkGHCJyGLNFwCf5ZwDw0nAici4GHCJymG3HKnCu9jJ6+ihx/3BeGk5EzsOAQ0QO8+8fTgMA/mtkP3gpFSJXQ0TuhAGHiBziWJkOe369AIVchrRREWKXQ0RuhgGHiByi5bYMkweHIKynt7jFEJHbYcAhoi534ZIBmwt4aTgRiYcBh4i63IZ9xdCbLIjr44+EiF5il0NEbogBh4i6lNFswZr8swCA2clRvDSciETBgENEXSr3aAXK6psQ6OuJqUNDxS6HiNwUAw4RdRlBELDq+18BAI8k8tJwIhIPAw4RdZlvjlbgYHEdVB5yPMJLw4lIRAw4RNQlDCYL/vvrYwCAeWP6Q+PvJXJFROTOGHCIqEus2XMWZ2oaEdRDhfl3DhC7HCJycww4RHTT6hoNeG/7CQDAC6m3oofKQ+SKiMjdMeAQ0U17b/tJ1F82YlCIH6YnhItdDhERAw4R3Zxfqy5idf4ZAMCiu2OgkHPeGyISHwMOEd2U//76OEwWAeMH9saY6N5il0NEBIABh4huwp5fa5D7cwUUchn+dFeM2OUQEVkx4BDRDbFYBPz1q58BAP81MhzRGj+RKyIi+g0DDhHdkM0FpThSqoOfygMLJ94qdjlERDYcGnBqa2uRlpYGtVoNtVqNtLQ01NXVdfgemUzW5uOtt96yjrnzzjtbvT5r1ixHbgoRXaXRYMJb3xQBAJ783S0I6qESuSIiIlsOnazi4Ycfxrlz57B161YAwBNPPIG0tDR8+eWX7b6nrKzM5vnXX3+NOXPm4KGHHrJZPm/ePCxdutT63NvbuwsrJ6KO/GPXaZTrmtC3lzdmJ0eKXQ4RUSsOCzjHjh3D1q1bsWfPHiQmJgIA/vGPfyApKQlFRUUYOHBgm+8LCQmxeb5lyxaMHz8e/fv3t1nu4+PTaiwROV6Frgkf7TwFAHhp8iDeUJOIuiWHBZz8/Hyo1WpruAGAUaNGQa1WIy8vr92Ac7WKigp89dVX+OSTT1q9tm7dOqxduxYajQZTpkzBa6+9Bj+/tk9y1Ov10Ov11uc6nQ4AYDQaYTQa7d00SWrpA/vhHK7c77e2HsdloxnDwtWYFBPkEtvgyv12Rey3c7lTv+3ZRocFnPLycgQHB7daHhwcjPLy8k6t45NPPoGfnx8efPBBm+WPPPIIoqKiEBISgiNHjiAjIwOHDh2CVqttcz2ZmZlYsmRJq+W5ubnw8fHpVC3uor0ekmO4Wr/PXQI2HVYAkGG8ugZff/212CXZxdX67erYb+dyh343NjZ2eqzdAWfx4sVthoWr7du3D0DzCcPXEgShzeVt+de//oVHHnkEXl62dyWeN2+e9fu4uDhER0cjISEBBw8exIgRI1qtJyMjA+np6dbnOp0O4eHhSE1Nhb+/f6dqkTqj0QitVouUlBQolUqxy5E8V+y3IAj4ffYBCLiAu4eEYMGMoWKX1Gmu2G9Xxn47lzv1u+UITGfYHXCeeuqp616xFBkZicOHD6OioqLVa1VVVdBoNNf9Obt370ZRURE2btx43bEjRoyAUqnEiRMn2gw4KpUKKlXrqzyUSqXkPwz2Yk+cy5X6nf3DaeT/egGeHnK8PCXGZeq+miv1WwrYb+dyh37bs312B5ygoCAEBQVdd1xSUhLq6+vx448/YuTIkQCAvXv3or6+HsnJydd9/6pVqxAfH4/bbrvtumOPHj0Ko9GI0NDQ628AEdmtsKQOr+ccA9B8YnF4AA/tElH35rB5cGJiYjB58mTMmzcPe/bswZ49ezBv3jxMnTrV5gTjQYMGYfPmzTbv1el0+OyzzzB37txW6z116hSWLl2K/fv348yZM8jJycH06dMxfPhwjB492lGbQ+S26hoNeHLdQRjNAqbEheAPoyPFLomI6LocOtHfunXrMGTIEKSmpiI1NRVDhw7FmjVrbMYUFRWhvr7eZtmGDRsgCAL+67/+q9U6PT09sX37dkyaNAkDBw7EM888g9TUVGzbtg0KBS9XJepKFouA5//nEErrLiMi0AdvTBva6XPoiIjE5NCJ/gICArB27doOxwiC0GrZE088gSeeeKLN8eHh4di5c2eX1EdEHft416/YfrwSnh5yrHhkBPy9pH18n4ikg/eiIqI27f21Bm/nNt+OYcm9gzE4TC1yRUREnceAQ0StVDXo8fT6ApgtAh4Y3gezbg8XuyQiIrsw4BCRDbNFwLMbClDZoEd0cA+8/kAcz7shIpfDgENENt7dfgJ5p2rgrVRgxSMj4OPp0FP1iIgcggGHiKx2/VKF5d+eAABkPjgE0Zq27+9GRNTdMeAQEQCgrP4yFm4shCAADyf2w/3D+4hdEhHRDWPAISIYTBY8/WkBLlwyYHCYP/48NVbskoiIbgoDDpGbazKa8cSa/dh/thZ+Kg+seGQEvJScNJOIXBvPHiRyYw1NRsz9ZD/2nr4AL6UcKx4dgYhAX7HLIiK6aQw4RG6q9pIBs//9Iw6dq4efygOrZt+OkVEBYpdFRNQlGHCI3FClrglpq35EUUUDevkosfoPiRjSlzMVE5F0MOAQuZmSC414dNVenK1phMZfhbVzEnk5OBFJDgMOkRs5VXURj/5zL8rqmxAe4I11c0ahX6CP2GUREXU5BhwiN3H0fD0eW/Ujai4ZcEtwD6ydk4gQtZfYZREROQQDDpEbOHC2FrP//SMamkwYHOaP1X8YicAeKrHLIiJyGAYcIgkTBAH/d7gML206jEaDGQkRvfCvx2+Hv5dS7NKIiByKAYdIoqoa9Hj18yPYerQcADAmOggfp8Xz5plE5Bb4l45IYgRBwBeHzuO1L46irtEID7kMT46/BU+OvwWeHpy8nIjcAwMOkYRU6pqw6PMj0P5cAQCIDfXHW9OHYnAY57ghIvfCgEMkAYIg4PPCUiz+4mfUXzZCqZDh6d9F4/+7cwCUCu61ISL3w4BD5OIqdE1YtPknbDtWCQCI6+OPt6bdhphQf5ErIyISDwMOkYu6pDdh/Y/FeG/7CeiaTFAqZFg48VY8MbY/99oQkdtjwCFyMZUNTfgk7wzW7ilG/WUjAGBoXzXemnYbBobwlgtERAADDpHLOFV1Ef/c/Ss2HSyFwWQBAEQF+eKJsf0xPb4vPLjXhojIigGHqJs7cLYWH+88Be2xCghC87Lh/Xrij2MHICVWA4VcJm6BRETdEAMOUTfU0GTEd0VVWJN/BvvO1FqXT4wJxh/HDUBCRC/IZAw2RETtYcAh6iYqG5qw7edKfHO0HHmnqmE0N++u8VTIcf/wMDwxtj9uCeY5NkREncGAQySiszWN+PaXanxztAIHi2uth6AAoH+QL+4aEoq0pAho/HnXbyIiezDgEDlRha4JhSV1OHCmBl8WKlCW/73N67eF90RqrAaTBmu4t4aI6CYw4BA5yCW9CT+V1qOwpA6HSupQWFKHsvqmq0bI4CGXYVT/QEwarMHEWA1C1d6i1UtEJCUMOEQ3qcloxtmaRpyuvoQzNZfwa9VFHD5Xj18qGmARbMfKZcCtGj8M7eMPz/piLJwxEUH+PuIUTkQkYQ4NOK+//jq++uorFBYWwtPTE3V1ddd9jyAIWLJkCVauXIna2lokJibigw8+wODBg61j9Ho9XnjhBaxfvx6XL1/GhAkTsGLFCvTt29eBW0PuymS2oPqiAZUNTSivb2oOMzWXcKa6+XHeZq+MrVC1F4aF98Rt4T0xLLwnhvRRw1flAaPRiJycs1B7K524JURE7sOhAcdgMGD69OlISkrCqlWrOvWeN998E++88w6ys7Nx66234q9//StSUlJQVFQEP7/mcxIWLlyIL7/8Ehs2bEBgYCCef/55TJ06FQcOHIBCoXDkJpEEGM0W6C4bUX/No67RiMqGJlTq9KhsaH5UNTSh5pLB5uTftvh7eSAqyBeRQb6IDPRFbJg/hoX35MnBREQicWjAWbJkCQAgOzu7U+MFQUBWVhYWLVqEBx98EADwySefQKPR4NNPP8Uf//hH1NfXY9WqVVizZg0mTpwIAFi7di3Cw8Oxbds2TJo0ySHbQo5lsQgwWiwwmQWYzL99bzBZYDCb0WS0QG+ywGCyQG8yX/W9BZeNZlw2mHBJb8ZloxmNBhMa9WY0GsxoNJrRqDfhot5kDTWXDGa765PLgKAeKmj8vdAv0AdRgc1hJirIB5GBvgjw9eS8NERE3Ui3Ogfn9OnTKC8vR2pqqnWZSqXCuHHjkJeXhz/+8Y84cOAAjEajzZiwsDDExcUhLy+vzYCj1+uh1+utz3U6HQDAaDTCaDR2Wf01F/VYsfO0zbJ2/8f/ql0CQtuLIVx5pWVZ2+MEm9cFwfZ9wpVvWr5vef2315qfmy0WVJTL8VVdAQQ0/0NtufI+iyBcea8Ai/DbcrNFgOWqZRZBgMXy23Kz5cpDEGCxCDBZmse2LLcGmivPnc1XpYDaSwl/byXU3h5QeyvRu4cKvf1UCPbzRG8/FXr3UCHYT4UAX88OZww2mUx2/eyWz11Xfv6ofey3c7HfzuVO/bZnG7tVwCkvLwcAaDQam+UajQZnz561jvH09ESvXr1ajWl5/7UyMzOte5OulpubCx+frjvBs/IysLqwW7XUTnLgQpXYRVjJZQI8ZIBSDnjIAA958+O3582veyoAlbz5q6e85blgs1ylAHwUArw9AB8PwNsDUMhMAPStf3Bj86OxAjiL5oejaLVaB66drsV+Oxf77Vzu0O/GxsZOj7X7X+PFixe3GRautm/fPiQkJNi7aqtrd/ULgnDd3f8djcnIyEB6err1uU6nQ3h4OFJTU+Hv73/DdV7rwiUDLqhb/3MoQ9t1XV2urN3lMpsBtuNkNstksqu/l101ruU1mXWM9b0yQC6TwWI2o6joOGJjYqBQKCCXya681vwT5LLfxjY/YB2jkMuuGi+zPveQyyCXAx5yOeSyK1/laF4uk0GpkMND0TzOQyGHUm67TMqHfIxGI7RaLVJSUqBU8kRjR2O/nYv9di536nfLEZjOsDvgPPXUU5g1a1aHYyIjI+1dLQAgJCQEQPNemtDQUOvyyspK616dkJAQGAwG1NbW2uzFqaysRHJycpvrValUUKlUrZYrlcou/TBoeirx0pTYLlufMxmNRuTUH8NdSZGS/wXpTrr6M0gdY7+di/12Lnfotz3bZ3fACQoKQlBQkL1v65SoqCiEhIRAq9Vi+PDhAJqvxNq5cyfeeOMNAEB8fDyUSiW0Wi1mzJgBACgrK8ORI0fw5ptvOqQuIiIici0OPWGkuLgYFy5cQHFxMcxmMwoLCwEAt9xyC3r06AEAGDRoEDIzM/HAAw9AJpNh4cKF+Nvf/obo6GhER0fjb3/7G3x8fPDwww8DANRqNebMmYPnn38egYGBCAgIwAsvvIAhQ4ZYr6oiIiIi9+bQgPPnP/8Zn3zyifV5y16Z7777DnfeeScAoKioCPX19dYxL774Ii5fvowFCxZYJ/rLzc21zoEDAMuWLYOHhwdmzJhhnegvOzubc+AQERERAAcHnOzs7OvOgSNcM4OaTCbD4sWLsXjx4nbf4+XlheXLl2P58uVdUCURERFJjVzsAoiIiIi6GgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUmOQ2cy7q5aZk+257brUmc0GtHY2AidTif5u9F2B+y3c7HfzsV+O5c79bvl3+1r74LQFrcMOA0NDQCA8PBwkSshIiIiezU0NECtVnc4RiZ0JgZJjMViwfnz5+Hn5weZTCZ2Od2CTqdDeHg4SkpK4O/vL3Y5ksd+Oxf77Vzst3O5U78FQUBDQwPCwsIgl3d8lo1b7sGRy+Xo27ev2GV0S/7+/pL/BelO2G/nYr+di/12Lnfp9/X23LTgScZEREQkOQw4REREJDkMOAQAUKlUeO2116BSqcQuxS2w387FfjsX++1c7Hfb3PIkYyIiIpI27sEhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAoXbp9XoMGzYMMpkMhYWFYpcjSWfOnMGcOXMQFRUFb29vDBgwAK+99hoMBoPYpUnGihUrEBUVBS8vL8THx2P37t1ilyRZmZmZuP322+Hn54fg4GDcf//9KCoqErsst5CZmQmZTIaFCxeKXUq3wYBD7XrxxRcRFhYmdhmSdvz4cVgsFnz88cc4evQoli1bho8++gh/+tOfxC5NEjZu3IiFCxdi0aJFKCgowJgxYzBlyhQUFxeLXZok7dy5E08++ST27NkDrVYLk8mE1NRUXLp0SezSJG3fvn1YuXIlhg4dKnYp3QovE6c2ff3110hPT8emTZswePBgFBQUYNiwYWKX5RbeeustfPjhh/j111/FLsXlJSYmYsSIEfjwww+ty2JiYnD//fcjMzNTxMrcQ1VVFYKDg7Fz506MHTtW7HIk6eLFixgxYgRWrFiBv/71rxg2bBiysrLELqtb4B4caqWiogLz5s3DmjVr4OPjI3Y5bqe+vh4BAQFil+HyDAYDDhw4gNTUVJvlqampyMvLE6kq91JfXw8A/Dw70JNPPom7774bEydOFLuUbsctb7ZJ7RMEAbNnz8b8+fORkJCAM2fOiF2SWzl16hSWL1+Ov//972KX4vKqq6thNpuh0Whslms0GpSXl4tUlfsQBAHp6em44447EBcXJ3Y5krRhwwYcPHgQ+/btE7uUbol7cNzE4sWLIZPJOnzs378fy5cvh06nQ0ZGhtglu7TO9vtq58+fx+TJkzF9+nTMnTtXpMqlRyaT2TwXBKHVMup6Tz31FA4fPoz169eLXYoklZSU4Nlnn8XatWvh5eUldjndEs/BcRPV1dWorq7ucExkZCRmzZqFL7/80uYfALPZDIVCgUceeQSffPKJo0uVhM72u+UP0/nz5zF+/HgkJiYiOzsbcjn/3+NmGQwG+Pj44LPPPsMDDzxgXf7ss8+isLAQO3fuFLE6aXv66afx+eefY9euXYiKihK7HEn6/PPP8cADD0ChUFiXmc1myGQyyOVy6PV6m9fcEQMO2SguLoZOp7M+P3/+PCZNmoT//d//RWJiIvr27StiddJUWlqK8ePHIz4+HmvXrnX7P0pdKTExEfHx8VixYoV1WWxsLO677z6eZOwAgiDg6aefxubNm7Fjxw5ER0eLXZJkNTQ04OzZszbLHn/8cQwaNAgvvfQSDwuC5+DQNfr162fzvEePHgCAAQMGMNw4wPnz53HnnXeiX79+ePvtt1FVVWV9LSQkRMTKpCE9PR1paWlISEhAUlISVq5cieLiYsyfP1/s0iTpySefxKeffootW7bAz8/Peq6TWq2Gt7e3yNVJi5+fX6sQ4+vri8DAQIabKxhwiESUm5uLkydP4uTJk60CJHeu3ryZM2eipqYGS5cuRVlZGeLi4pCTk4OIiAixS5Oklsvx77zzTpvl//73vzF79mznF0RujYeoiIiISHJ4JiMRERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUnO/wN9DLF6gnWttQAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(-5, 5, 0.2), np.tanh(np.arange(-5, 5, 0.2))); plt.grid();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add `tanh`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        return Value(self.data * other.data, (self, other), '*')\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inputs: x1, x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weights: w1, w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = Value(6.8813735870195432, label='b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### x1w1 + x2w2 + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o = n.tanh(); o.label = 'o'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing gradient of each node manually"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial o}{\\partial o} = 1 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o.grad = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ o = \\tanh(n) $$\n",
    "$$ \\frac{\\partial o}{\\partial n} = \\frac{\\partial{\\tanh(n)}}{\\partial n} = 1 - \\tanh(n)^2 = 1 - o^2 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.4999999999999999\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "1 - (o.data ** 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n.grad = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> With pluses as we saw the gradient will be same as previous gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1w1x2w2.grad = 0.5\n",
    "b.grad = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1w1.grad = 0.5\n",
    "x2w2.grad = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x2.grad = w2.data * x2w2.grad\n",
    "w2.grad = x2.data * x2w2.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1.grad = w1.data * x1w1.grad\n",
    "w1.grad = x1.data * x1w1.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045179408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045179408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-54.5 1382,-90.5 1567,-90.5 1567,-54.5 1382,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-54.5 1405,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-54.5 1485,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045179408tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045179408tanh&#45;&gt;140404045179408 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045179408tanh&#45;&gt;140404045179408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-72.5C1353.58,-72.5 1362.3,-72.5 1371.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-76 1381.81,-72.5 1371.81,-69 1371.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045179920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179920&#45;&gt;140404045179408tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045179920&#45;&gt;140404045179408tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-72.5C1265.01,-72.5 1273.74,-72.5 1281.66,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-76 1291.91,-72.5 1281.91,-69 1281.91,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179920+ -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045179920+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179920+&#45;&gt;140404045179920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045179920+&#45;&gt;140404045179920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045179856 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045179856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179856&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045179856&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045179856+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045179856+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045179856+&#45;&gt;140404045179856 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045179856+&#45;&gt;140404045179856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404043109136 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404043109136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045179792* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045179792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404043109136&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404043109136&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045179728 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045179728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179728&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045179728&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045179728* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045179728*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045179728*&#45;&gt;140404045179728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045179728*&#45;&gt;140404045179728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404038424400 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404038424400</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-165.5 2,-201.5 198,-201.5 198,-165.5 2,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-165.5 32,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-165.5 112,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140404038424400&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404038424400&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045179792 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045179792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045179792&#45;&gt;140404045179856+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045179792&#45;&gt;140404045179856+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045179792*&#45;&gt;140404045179792 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045179792*&#45;&gt;140404045179792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404043109328 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404043109328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404043109328&#45;&gt;140404045179728* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404043109328&#45;&gt;140404045179728*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404042677648 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404042677648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404042677648&#45;&gt;140404045179792* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404042677648&#45;&gt;140404045179792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404043109840 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404043109840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404043109840&#45;&gt;140404045179920+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404043109840&#45;&gt;140404045179920+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing gradient of each node with `_backward()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad = 1.0 * out.grad\n",
    "            other.grad = 1.0 * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad = other.data * out.grad\n",
    "            other.grad = self.data * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad = (1 - t**2) * out.grad\n",
    "        out._backward = _backward\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Lets take the NN code from the above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "\n",
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n'\n",
    "o = n.tanh(); o.label = 'o'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404041050640 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404041050640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-55.5 326,-91.5 546,-91.5 546,-55.5 326,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-55.5 379,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-55.5 464,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045193488+ -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045193488+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404041050640&#45;&gt;140404045193488+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404041050640&#45;&gt;140404045193488+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-90.75C555.64,-92.23 564.6,-93.65 572.65,-94.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.23,-98.4 582.65,-96.5 573.32,-91.48 572.23,-98.4\"/>\n",
       "</g>\n",
       "<!-- 140404041050640* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404041050640*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404041050640*&#45;&gt;140404041050640 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404041050640*&#45;&gt;140404041050640</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C297.77,-73.5 306.37,-73.5 315.6,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-77 325.84,-73.5 315.84,-70 315.84,-77\"/>\n",
       "</g>\n",
       "<!-- 140404041051152 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404041051152</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404041050832* -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404041050832*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404041051152&#45;&gt;140404041050832* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404041051152&#45;&gt;140404041050832*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404041051728 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404041051728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-55.5 4,-91.5 196,-91.5 196,-55.5 4,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-55.5 34,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-55.5 114,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404041051728&#45;&gt;140404041050640* -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404041051728&#45;&gt;140404041050640*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.05,-73.5C206.52,-73.5 216.65,-73.5 225.71,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.79,-77 235.79,-73.5 225.79,-70 225.79,-77\"/>\n",
       "</g>\n",
       "<!-- 140404041051792 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404041051792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045196624+ -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045196624+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404041051792&#45;&gt;140404045196624+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404041051792&#45;&gt;140404045196624+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140404041050832 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404041050832</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-110.5 328.5,-146.5 543.5,-146.5 543.5,-110.5 328.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-110.5 381.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-110.5 461.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404041050832&#45;&gt;140404045193488+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404041050832&#45;&gt;140404045193488+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-111.01C554.01,-109.34 563.76,-107.74 572.44,-106.32\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.17,-109.75 582.47,-104.68 572.04,-102.84 573.17,-109.75\"/>\n",
       "</g>\n",
       "<!-- 140404041050832*&#45;&gt;140404041050832 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404041050832*&#45;&gt;140404041050832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C298.51,-128.5 308.08,-128.5 318.36,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-132 328.39,-128.5 318.39,-125 318.39,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045193488 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045193488</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045193488&#45;&gt;140404045196624+ -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045193488&#45;&gt;140404045196624+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140404045193488+&#45;&gt;140404045193488 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045193488+&#45;&gt;140404045193488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404041050512 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404041050512</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-110.5 2.5,-146.5 197.5,-146.5 197.5,-110.5 2.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-110.5 35.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-110.5 115.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404041050512&#45;&gt;140404041050832* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404041050512&#45;&gt;140404041050832*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-128.5C207.65,-128.5 217.05,-128.5 225.52,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-132 235.7,-128.5 225.7,-125 225.7,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045196624 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045196624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045195152tanh -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045195152tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045196624&#45;&gt;140404045195152tanh -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045196624&#45;&gt;140404045195152tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045196624+&#45;&gt;140404045196624 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045196624+&#45;&gt;140404045196624</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045195152 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404045195152</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045195152tanh&#45;&gt;140404045195152 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045195152tanh&#45;&gt;140404045195152</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404029101008 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404029101008</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 200,-36.5 200,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-0.5 33,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-0.5 118,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404029101008&#45;&gt;140404041050640* -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404029101008&#45;&gt;140404041050640*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `o`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.4999999999999999\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "o.grad = 1.0 # setting this to 1 because Value's grad variable is 0\n",
    "o._backward();  n.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `n`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;36m0.4999999999999999\u001b[0m, \u001b[1;36m0.4999999999999999\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n._backward(); \n",
    "b.grad, x1w1x2w2.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `b`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b._backward();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `x1w1x2w2`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;36m0.4999999999999999\u001b[0m, \u001b[1;36m0.4999999999999999\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1w1x2w2._backward(); \n",
    "x1w1.grad, x2w2.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `x2w2`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4999999999999999</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;36m0.4999999999999999\u001b[0m, \u001b[1;36m0.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x2w2._backward()\n",
    "x2.grad, w2.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backward on `x1w1`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.4999999999999996</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9999999999999998</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[1;36m-1.4999999999999996\u001b[0m, \u001b[1;36m0.9999999999999998\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1w1._backward()\n",
    "x1.grad, w1.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Draw the computation graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404041050640 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404041050640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-55.5 326,-91.5 546,-91.5 546,-55.5 326,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-55.5 379,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-55.5 464,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045193488+ -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045193488+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404041050640&#45;&gt;140404045193488+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404041050640&#45;&gt;140404045193488+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-90.75C555.64,-92.23 564.6,-93.65 572.65,-94.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.23,-98.4 582.65,-96.5 573.32,-91.48 572.23,-98.4\"/>\n",
       "</g>\n",
       "<!-- 140404041050640* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404041050640*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404041050640*&#45;&gt;140404041050640 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404041050640*&#45;&gt;140404041050640</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C297.77,-73.5 306.37,-73.5 315.6,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-77 325.84,-73.5 315.84,-70 315.84,-77\"/>\n",
       "</g>\n",
       "<!-- 140404041051152 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404041051152</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404041050832* -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404041050832*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404041051152&#45;&gt;140404041050832* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404041051152&#45;&gt;140404041050832*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404041051728 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404041051728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-55.5 2,-91.5 198,-91.5 198,-55.5 2,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-55.5 32,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-55.5 112,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140404041051728&#45;&gt;140404041050640* -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404041051728&#45;&gt;140404041050640*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.37,-73.5C208.05,-73.5 217.4,-73.5 225.8,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.91,-77 235.91,-73.5 225.91,-70 225.91,-77\"/>\n",
       "</g>\n",
       "<!-- 140404041051792 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404041051792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045196624+ -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045196624+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404041051792&#45;&gt;140404045196624+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404041051792&#45;&gt;140404045196624+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140404041050832 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404041050832</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-110.5 328.5,-146.5 543.5,-146.5 543.5,-110.5 328.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-110.5 381.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-110.5 461.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404041050832&#45;&gt;140404045193488+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404041050832&#45;&gt;140404045193488+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-111.01C554.01,-109.34 563.76,-107.74 572.44,-106.32\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.17,-109.75 582.47,-104.68 572.04,-102.84 573.17,-109.75\"/>\n",
       "</g>\n",
       "<!-- 140404041050832*&#45;&gt;140404041050832 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404041050832*&#45;&gt;140404041050832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C298.51,-128.5 308.08,-128.5 318.36,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-132 328.39,-128.5 318.39,-125 318.39,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045193488 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045193488</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045193488&#45;&gt;140404045196624+ -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045193488&#45;&gt;140404045196624+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140404045193488+&#45;&gt;140404045193488 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045193488+&#45;&gt;140404045193488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404041050512 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404041050512</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-110.5 2.5,-146.5 197.5,-146.5 197.5,-110.5 2.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-110.5 35.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-110.5 115.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404041050512&#45;&gt;140404041050832* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404041050512&#45;&gt;140404041050832*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-128.5C207.65,-128.5 217.05,-128.5 225.52,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-132 235.7,-128.5 225.7,-125 225.7,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045196624 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045196624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045195152tanh -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045195152tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045196624&#45;&gt;140404045195152tanh -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045196624&#45;&gt;140404045195152tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045196624+&#45;&gt;140404045196624 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045196624+&#45;&gt;140404045196624</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045195152 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404045195152</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045195152tanh&#45;&gt;140404045195152 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045195152tanh&#45;&gt;140404045195152</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404029101008 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404029101008</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 200,-36.5 200,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-0.5 33,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-0.5 118,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404029101008&#45;&gt;140404041050640* -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404029101008&#45;&gt;140404041050640*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backward propogation with one call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "\n",
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n'\n",
    "o = n.tanh(); o.label = 'o'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045382672 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045382672</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045382608+ -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404045382608+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045382672&#45;&gt;140404045382608+ -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045382672&#45;&gt;140404045382608+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140404045382672+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045382672+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045382672+&#45;&gt;140404045382672 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045382672+&#45;&gt;140404045382672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404045389392 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045389392</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-55.5 4,-91.5 196,-91.5 196,-55.5 4,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-55.5 34,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-55.5 114,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045385040* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045385040*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045389392&#45;&gt;140404045385040* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404045389392&#45;&gt;140404045385040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.05,-73.5C206.52,-73.5 216.65,-73.5 225.71,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.79,-77 235.79,-73.5 225.79,-70 225.79,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045384848 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045384848</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384848tanh -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045384848tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045384848tanh&#45;&gt;140404045384848 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045384848tanh&#45;&gt;140404045384848</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045389456 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045389456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384976* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045384976*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045389456&#45;&gt;140404045384976* -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045389456&#45;&gt;140404045384976*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045389520 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045389520</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389520&#45;&gt;140404045384976* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045389520&#45;&gt;140404045384976*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045384976 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045384976</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384976&#45;&gt;140404045382672+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045384976&#45;&gt;140404045382672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045384976*&#45;&gt;140404045384976 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045384976*&#45;&gt;140404045384976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045389584 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404045389584</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389584&#45;&gt;140404045382608+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045389584&#45;&gt;140404045382608+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140404045385040 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045385040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045385040&#45;&gt;140404045382672+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404045385040&#45;&gt;140404045382672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045385040*&#45;&gt;140404045385040 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045385040*&#45;&gt;140404045385040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045389712 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404045389712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 197.5,-36.5 197.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-0.5 35.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-0.5 115.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389712&#45;&gt;140404045385040* -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404045389712&#45;&gt;140404045385040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "<!-- 140404045382608 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045382608</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045382608&#45;&gt;140404045384848tanh -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045382608&#45;&gt;140404045384848tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045382608+&#45;&gt;140404045382608 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045382608+&#45;&gt;140404045382608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Topological sort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">x1</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">w1</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-3.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">x1</span>*w1 <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-6.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">w2</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">x2</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">x2</span>*w2 <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">x1</span>*w1 + x2*w2 <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-6.0</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">b</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.881373587019543</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">n</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8813735870195432</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>=<span style=\"color: #800080; text-decoration-color: #800080\">o</span> <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7071067811865476</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mx1\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m2\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mw1\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m-3.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mx1\u001b[0m*w1 \u001b[33mdata\u001b[0m=\u001b[1;36m-6.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mw2\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m1\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mx2\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mx2\u001b[0m*w2 \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mx1\u001b[0m*w1 + x2*w2 \u001b[33mdata\u001b[0m=\u001b[1;36m-6.0\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mb\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m6\u001b[0m\u001b[1;36m.881373587019543\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mn\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8813735870195432\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m=\u001b[35mo\u001b[0m \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7071067811865476\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "topo = []\n",
    "visited = set()\n",
    "def build_topo(v):\n",
    "    if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "            build_topo(child)\n",
    "        topo.append(v)\n",
    "build_topo(o)\n",
    "topo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply `backward` in reverse order of topological order of the computation graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for node in reversed(topo):\n",
    "    node._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045382672 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045382672</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045382608+ -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404045382608+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045382672&#45;&gt;140404045382608+ -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045382672&#45;&gt;140404045382608+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140404045382672+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045382672+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045382672+&#45;&gt;140404045382672 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045382672+&#45;&gt;140404045382672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404045389392 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045389392</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-55.5 4,-91.5 196,-91.5 196,-55.5 4,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-55.5 34,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-55.5 114,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045385040* -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045385040*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045389392&#45;&gt;140404045385040* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404045389392&#45;&gt;140404045385040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.05,-73.5C206.52,-73.5 216.65,-73.5 225.71,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.79,-77 235.79,-73.5 225.79,-70 225.79,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045384848 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045384848</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384848tanh -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045384848tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045384848tanh&#45;&gt;140404045384848 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045384848tanh&#45;&gt;140404045384848</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045389456 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045389456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-165.5 2,-201.5 198,-201.5 198,-165.5 2,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-165.5 32,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-165.5 112,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384976* -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045384976*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045389456&#45;&gt;140404045384976* -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045389456&#45;&gt;140404045384976*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045389520 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045389520</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389520&#45;&gt;140404045384976* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045389520&#45;&gt;140404045384976*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045384976 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045384976</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045384976&#45;&gt;140404045382672+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045384976&#45;&gt;140404045382672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045384976*&#45;&gt;140404045384976 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045384976*&#45;&gt;140404045384976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045389584 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404045389584</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389584&#45;&gt;140404045382608+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045389584&#45;&gt;140404045382608+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140404045385040 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045385040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045385040&#45;&gt;140404045382672+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404045385040&#45;&gt;140404045382672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045385040*&#45;&gt;140404045385040 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045385040*&#45;&gt;140404045385040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045389712 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404045389712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 197.5,-36.5 197.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-0.5 35.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-0.5 115.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045389712&#45;&gt;140404045385040* -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404045389712&#45;&gt;140404045385040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "<!-- 140404045382608 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045382608</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045382608&#45;&gt;140404045384848tanh -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045382608&#45;&gt;140404045384848tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045382608+&#45;&gt;140404045382608 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045382608+&#45;&gt;140404045382608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add `backward` to `Value`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad = 1.0 * out.grad\n",
    "            other.grad = 1.0 * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad = other.data * out.grad\n",
    "            other.grad = self.data * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad = (1 - t**2) * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "\n",
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n'\n",
    "o = n.tanh(); o.label = 'o'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140404045396496 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045396496</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045387472+ -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404045387472+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045396496&#45;&gt;140404045387472+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045396496&#45;&gt;140404045387472+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045396496* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045396496*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045396496*&#45;&gt;140404045396496 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045396496*&#45;&gt;140404045396496</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045388304 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045388304</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045388304tanh -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045388304tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404045388304tanh&#45;&gt;140404045388304 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045388304tanh&#45;&gt;140404045388304</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045386832 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045386832</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045386832&#45;&gt;140404045388304tanh -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045386832&#45;&gt;140404045388304tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045386832+ -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045386832+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045386832+&#45;&gt;140404045386832 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045386832+&#45;&gt;140404045386832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140404045397584 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045397584</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045397584&#45;&gt;140404045387472+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404045397584&#45;&gt;140404045387472+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045397584* -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045397584*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045397584*&#45;&gt;140404045397584 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045397584*&#45;&gt;140404045397584</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045387472 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045387472</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045387472&#45;&gt;140404045386832+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045387472&#45;&gt;140404045386832+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140404045387472+&#45;&gt;140404045387472 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045387472+&#45;&gt;140404045387472</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404045396752 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045396752</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045396752&#45;&gt;140404045386832+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045396752&#45;&gt;140404045386832+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140404045395216 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045395216</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 197.5,-91.5 197.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-55.5 35.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-55.5 115.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045395216&#45;&gt;140404045396496* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045395216&#45;&gt;140404045396496*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-73.5C207.65,-73.5 217.05,-73.5 225.52,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-77 235.7,-73.5 225.7,-70 225.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404033533712 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404033533712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-165.5 0,-201.5 200,-201.5 200,-165.5 0,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-165.5 33,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-165.5 118,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404033533712&#45;&gt;140404045397584* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404033533712&#45;&gt;140404045397584*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045394768 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045394768</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-110.5 2,-146.5 198,-146.5 198,-110.5 2,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-110.5 32,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-110.5 112,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140404045394768&#45;&gt;140404045397584* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404045394768&#45;&gt;140404045397584*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.37,-128.5C208.05,-128.5 217.4,-128.5 225.8,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.91,-132 235.91,-128.5 225.91,-125 225.91,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045396944 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404045396944</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 196,-36.5 196,-0.5 4,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-0.5 34,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-0.5 114,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045396944&#45;&gt;140404045396496* -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045396944&#45;&gt;140404045396496*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fixing a backprop bug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"504pt\" height=\"45pt\"\n",
       " viewBox=\"0.00 0.00 504.00 45.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 41)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-41 500,-41 500,4 -4,4\"/>\n",
       "<!-- 140404045374800 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045374800</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 185,-36.5 185,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-0.5 23,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"63\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"103,-0.5 103,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045226192+ -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045226192+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"248\" cy=\"-18.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"248\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045374800&#45;&gt;140404045226192+ -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045374800&#45;&gt;140404045226192+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M185.01,-18.5C194.01,-18.5 202.74,-18.5 210.66,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"210.91,-22 220.91,-18.5 210.91,-15 210.91,-22\"/>\n",
       "</g>\n",
       "<!-- 140404045226192 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045226192</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"311,-0.5 311,-36.5 496,-36.5 496,-0.5 311,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"322.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"334,-0.5 334,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"374\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"414,-0.5 414,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"455\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045226192+&#45;&gt;140404045226192 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045226192+&#45;&gt;140404045226192</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M275.04,-18.5C282.58,-18.5 291.3,-18.5 300.57,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.81,-22 310.81,-18.5 300.81,-15 300.81,-22\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(3.0, label='b')\n",
    "b = a + a; b.label = 'b'\n",
    "b.backward()\n",
    "draw_dot(b, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"832pt\" height=\"100pt\"\n",
       " viewBox=\"0.00 0.00 832.00 100.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 96)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-96 828,-96 828,4 -4,4\"/>\n",
       "<!-- 140404045541456 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045541456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-55.5 2.5,-91.5 191.5,-91.5 191.5,-55.5 2.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-55.5 25.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-55.5 105.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"148.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404045541968+ -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045541968+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"257\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045541456&#45;&gt;140404045541968+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045541456&#45;&gt;140404045541968+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M191.74,-73.5C201.55,-73.5 211.04,-73.5 219.59,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"219.87,-77 229.87,-73.5 219.87,-70 219.87,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045541712* -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045541712*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"257\" cy=\"-18.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045541456&#45;&gt;140404045541712* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045541456&#45;&gt;140404045541712*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M166.84,-55.49C176.04,-52.69 185.3,-49.67 194,-46.5 204.51,-42.67 215.73,-37.84 225.7,-33.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"227.23,-36.43 234.81,-29.02 224.27,-30.09 227.23,-36.43\"/>\n",
       "</g>\n",
       "<!-- 140404045541968 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045541968</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"320.5,-55.5 320.5,-91.5 509.5,-91.5 509.5,-55.5 320.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"332\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"343.5,-55.5 343.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"423.5,-55.5 423.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404045542224* -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045542224*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"573\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"573\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045541968&#45;&gt;140404045542224* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404045541968&#45;&gt;140404045542224*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M509.9,-56.65C519.42,-54.94 528.62,-53.29 536.89,-51.8\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"537.59,-55.23 546.81,-50.02 536.35,-48.34 537.59,-55.23\"/>\n",
       "</g>\n",
       "<!-- 140404045541968+&#45;&gt;140404045541968 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045541968+&#45;&gt;140404045541968</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M284.11,-73.5C291.84,-73.5 300.8,-73.5 310.35,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"310.48,-77 320.48,-73.5 310.48,-70 310.48,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045541520 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045541520</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 194,-36.5 194,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-0.5 23,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-0.5 108,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"151\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404045541520&#45;&gt;140404045541968+ -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045541520&#45;&gt;140404045541968+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.94,-36.53C174.08,-39.65 184.37,-43.01 194,-46.5 204.27,-50.22 215.26,-54.8 225.09,-59.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.91,-62.43 234.47,-63.32 226.77,-56.04 223.91,-62.43\"/>\n",
       "</g>\n",
       "<!-- 140404045541520&#45;&gt;140404045541712* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045541520&#45;&gt;140404045541712*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M194.01,-18.5C202.98,-18.5 211.65,-18.5 219.51,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"219.7,-22 229.7,-18.5 219.7,-15 219.7,-22\"/>\n",
       "</g>\n",
       "<!-- 140404045541712 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045541712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"320,-0.5 320,-36.5 510,-36.5 510,-0.5 320,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"331.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"343,-0.5 343,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"385.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"428,-0.5 428,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"469\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045541712&#45;&gt;140404045542224* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045541712&#45;&gt;140404045542224*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M510.34,-34.83C519.62,-36.44 528.58,-37.99 536.66,-39.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"536.27,-42.87 546.72,-41.12 537.46,-35.97 536.27,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404045541712*&#45;&gt;140404045541712 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045541712*&#45;&gt;140404045541712</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M284.11,-18.5C291.63,-18.5 300.32,-18.5 309.58,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"309.81,-22 319.81,-18.5 309.81,-15 309.81,-22\"/>\n",
       "</g>\n",
       "<!-- 140404045542224 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045542224</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"636,-27.5 636,-63.5 824,-63.5 824,-27.5 636,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"646.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"657,-27.5 657,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"699.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"742,-27.5 742,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"783\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045542224*&#45;&gt;140404045542224 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045542224*&#45;&gt;140404045542224</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M600.29,-45.5C607.78,-45.5 616.42,-45.5 625.6,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.76,-49 635.76,-45.5 625.76,-42 625.76,-49\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(-2.0, label='a')\n",
    "b = Value(3.0, label='b')\n",
    "d = a*b; d.label = 'd'\n",
    "e = a+b; e.label = 'e'\n",
    "f = d*e; f.label = 'f'\n",
    "f.backward()\n",
    "draw_dot(f, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accumulate the gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "            other.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad # <- Accumulate the gradient\n",
    "            other.grad += self.data * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"504pt\" height=\"45pt\"\n",
       " viewBox=\"0.00 0.00 504.00 45.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 41)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-41 500,-41 500,4 -4,4\"/>\n",
       "<!-- 140404045229456 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045229456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"311,-0.5 311,-36.5 496,-36.5 496,-0.5 311,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"322.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"334,-0.5 334,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"374\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"414,-0.5 414,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"455\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045229456+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045229456+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"248\" cy=\"-18.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"248\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045229456+&#45;&gt;140404045229456 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045229456+&#45;&gt;140404045229456</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M275.04,-18.5C282.58,-18.5 291.3,-18.5 300.57,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.81,-22 310.81,-18.5 300.81,-15 300.81,-22\"/>\n",
       "</g>\n",
       "<!-- 140404045229328 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045229328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 185,-36.5 185,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-0.5 23,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"63\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"103,-0.5 103,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 2.0000</text>\n",
       "</g>\n",
       "<!-- 140404045229328&#45;&gt;140404045229456+ -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045229328&#45;&gt;140404045229456+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M185.01,-18.5C194.01,-18.5 202.74,-18.5 210.66,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"210.91,-22 220.91,-18.5 210.91,-15 210.91,-22\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(3.0, label='b')\n",
    "b = a + a; b.label = 'b'\n",
    "b.backward()\n",
    "draw_dot(b, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"832pt\" height=\"100pt\"\n",
       " viewBox=\"0.00 0.00 832.00 100.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 96)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-96 828,-96 828,4 -4,4\"/>\n",
       "<!-- 140404045543440 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045543440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"320.5,-55.5 320.5,-91.5 509.5,-91.5 509.5,-55.5 320.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"332\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"343.5,-55.5 343.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"423.5,-55.5 423.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;6.0000</text>\n",
       "</g>\n",
       "<!-- 140404045543888* -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045543888*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"573\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"573\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045543440&#45;&gt;140404045543888* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045543440&#45;&gt;140404045543888*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M509.9,-56.65C519.42,-54.94 528.62,-53.29 536.89,-51.8\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"537.59,-55.23 546.81,-50.02 536.35,-48.34 537.59,-55.23\"/>\n",
       "</g>\n",
       "<!-- 140404045543440+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045543440+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"257\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045543440+&#45;&gt;140404045543440 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045543440+&#45;&gt;140404045543440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M284.11,-73.5C291.84,-73.5 300.8,-73.5 310.35,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"310.48,-77 320.48,-73.5 310.48,-70 310.48,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045377488 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045377488</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 194,-91.5 194,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"11.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">a</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"23,-55.5 23,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108,-55.5 108,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"151\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 140404045377488&#45;&gt;140404045543440+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045377488&#45;&gt;140404045543440+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M194.01,-73.5C202.98,-73.5 211.65,-73.5 219.51,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"219.7,-77 229.7,-73.5 219.7,-70 219.7,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045544144* -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045544144*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"257\" cy=\"-18.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045377488&#45;&gt;140404045544144* -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045377488&#45;&gt;140404045544144*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M166.84,-55.49C176.04,-52.69 185.3,-49.67 194,-46.5 204.51,-42.67 215.73,-37.84 225.7,-33.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"227.23,-36.43 234.81,-29.02 224.27,-30.09 227.23,-36.43\"/>\n",
       "</g>\n",
       "<!-- 140404045544144 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045544144</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"320,-0.5 320,-36.5 510,-36.5 510,-0.5 320,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"331.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">d</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"343,-0.5 343,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"385.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"428,-0.5 428,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"469\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045544144&#45;&gt;140404045543888* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045544144&#45;&gt;140404045543888*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M510.34,-34.83C519.62,-36.44 528.58,-37.99 536.66,-39.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"536.27,-42.87 546.72,-41.12 537.46,-35.97 536.27,-42.87\"/>\n",
       "</g>\n",
       "<!-- 140404045544144*&#45;&gt;140404045544144 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045544144*&#45;&gt;140404045544144</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M284.11,-18.5C291.63,-18.5 300.32,-18.5 309.58,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"309.81,-22 319.81,-18.5 309.81,-15 309.81,-22\"/>\n",
       "</g>\n",
       "<!-- 140404045543888 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045543888</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"636,-27.5 636,-63.5 824,-63.5 824,-27.5 636,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"646.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">f</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"657,-27.5 657,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"699.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"742,-27.5 742,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"783\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045543888*&#45;&gt;140404045543888 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045543888*&#45;&gt;140404045543888</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M600.29,-45.5C607.78,-45.5 616.42,-45.5 625.6,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.76,-49 635.76,-45.5 625.76,-42 625.76,-49\"/>\n",
       "</g>\n",
       "<!-- 140404045376976 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045376976</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 191.5,-36.5 191.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"25.5,-0.5 25.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"65.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105.5,-0.5 105.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"148.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;8.0000</text>\n",
       "</g>\n",
       "<!-- 140404045376976&#45;&gt;140404045543440+ -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045376976&#45;&gt;140404045543440+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.94,-36.53C174.08,-39.65 184.37,-43.01 194,-46.5 204.27,-50.22 215.26,-54.8 225.09,-59.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.91,-62.43 234.47,-63.32 226.77,-56.04 223.91,-62.43\"/>\n",
       "</g>\n",
       "<!-- 140404045376976&#45;&gt;140404045544144* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404045376976&#45;&gt;140404045544144*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M191.74,-18.5C201.55,-18.5 211.04,-18.5 219.59,-18.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"219.87,-22 229.87,-18.5 219.87,-15 219.87,-22\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(-2.0, label='a')\n",
    "b = Value(3.0, label='b')\n",
    "d = a*b; d.label = 'd'\n",
    "e = a+b; e.label = 'e'\n",
    "f = d*e; f.label = 'f'\n",
    "f.backward()\n",
    "draw_dot(f, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add and multiply `Value` object with constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "            other.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def __radd__(self, other): # other + self\n",
    "        return self + other\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad # <- Accumulate the gradient\n",
    "            other.grad += self.data * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def __rmul__(self, other): # other * self\n",
    "        return self * other\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m3\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0); a + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m2\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = Value(2.0); a * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m4\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "2 * a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m4\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "2 + a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implement `tanh`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0 # ⭠ Add grad\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(label={self.label} data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "            other.grad += 1.0 * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def __radd__(self, other): # other + self\n",
    "        return self + other\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad # <- Accumulate the gradient\n",
    "            other.grad += self.data * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def __rmul__(self, other): # other * self\n",
    "        return self * other\n",
    "    \n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "        out = Value(self.data ** other, (self,), f'**{other}')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += other * (self.data ** (other - 1)) * out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "        \n",
    "    \n",
    "    def exp(self):\n",
    "        x = self.data\n",
    "        out = Value(math.exp(x), (self,), 'exp')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += out.data * out.grad\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "    \n",
    "    def __truediv__(self, other): # self / other\n",
    "        return self * other**-1\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad # <- Accumulate the gradient\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def __neg__(self): #-self\n",
    "        return -self\n",
    "    \n",
    "    def __sub__(self, other): # self - other\n",
    "        return self + (-other)\n",
    "        \n",
    "    def backward(self):\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Value(2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7.38905609893065</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m7\u001b[0m\u001b[1;36m.38905609893065\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a.exp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = Value(3.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6666666666666666</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6666666666666666\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a/b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m16\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a **4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m1\u001b[0m\u001b[1;36m.0\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"2944pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 2944.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 2940,-206 2940,4 -4,4\"/>\n",
       "<!-- 140404045670416 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404045670416</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2225,-108.5 2225,-144.5 2407,-144.5 2407,-108.5 2225,-108.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2235\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2245,-108.5 2245,-144.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2285\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2325,-108.5 2325,-144.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2366\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n",
       "</g>\n",
       "<!-- 140404045524432* -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140404045524432*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2688\" cy=\"-98.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2688\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045670416&#45;&gt;140404045524432* -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140404045670416&#45;&gt;140404045524432*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2407.31,-119.9C2468.7,-115.37 2551.76,-109.19 2625,-103.5 2633.42,-102.85 2642.48,-102.13 2651.01,-101.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2651.37,-104.92 2661.06,-100.63 2650.81,-97.94 2651.37,-104.92\"/>\n",
       "</g>\n",
       "<!-- 140404045670416+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404045670416+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1940\" cy=\"-126.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1940\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045670416+&#45;&gt;140404045670416 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404045670416+&#45;&gt;140404045670416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1967,-126.5C2017.46,-126.5 2131.06,-126.5 2214.81,-126.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2214.87,-130 2224.87,-126.5 2214.87,-123 2214.87,-130\"/>\n",
       "</g>\n",
       "<!-- 140404045669904 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404045669904</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-81.5 1382,-117.5 1564,-117.5 1564,-81.5 1382,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1392\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1402,-81.5 1402,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1442\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.7627</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1482,-81.5 1482,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1523\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.2500</text>\n",
       "</g>\n",
       "<!-- 140404045670096exp -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140404045670096exp</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1627\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1627\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">exp</text>\n",
       "</g>\n",
       "<!-- 140404045669904&#45;&gt;140404045670096exp -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140404045669904&#45;&gt;140404045670096exp</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1564.18,-99.5C1573.14,-99.5 1581.83,-99.5 1589.73,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1589.96,-103 1599.96,-99.5 1589.96,-96 1589.96,-103\"/>\n",
       "</g>\n",
       "<!-- 140404045669904* -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404045669904*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045669904*&#45;&gt;140404045669904 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404045669904*&#45;&gt;140404045669904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.13,-99.5C1353.67,-99.5 1362.39,-99.5 1371.64,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.87,-103 1381.87,-99.5 1371.87,-96 1371.87,-103\"/>\n",
       "</g>\n",
       "<!-- 140404045663760 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404045663760</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-55.5 4,-91.5 196,-91.5 196,-55.5 4,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-55.5 34,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-55.5 114,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045669008* -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404045669008*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045663760&#45;&gt;140404045669008* -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140404045663760&#45;&gt;140404045669008*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.05,-73.5C206.52,-73.5 216.65,-73.5 225.71,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.79,-77 235.79,-73.5 225.79,-70 225.79,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045669840 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404045669840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1072.5,-109.5 1072.5,-145.5 1254.5,-145.5 1254.5,-109.5 1072.5,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1092.5,-109.5 1092.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1132.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1172.5,-109.5 1172.5,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1213.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.2203</text>\n",
       "</g>\n",
       "<!-- 140404045669840&#45;&gt;140404045669904* -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140404045669840&#45;&gt;140404045669904*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1254.68,-111.05C1264.61,-109.24 1274.23,-107.48 1282.84,-105.91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1283.6,-109.33 1292.81,-104.09 1282.35,-102.45 1283.6,-109.33\"/>\n",
       "</g>\n",
       "<!-- 140404045663824 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404045663824</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-165.5 2,-201.5 198,-201.5 198,-165.5 2,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-165.5 32,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-165.5 112,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140404045668496* -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404045668496*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045663824&#45;&gt;140404045668496* -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404045663824&#45;&gt;140404045668496*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140404045670992 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404045670992</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2443,-57.5 2443,-93.5 2625,-93.5 2625,-57.5 2443,-57.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2453\" y=\"-71.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2463,-57.5 2463,-93.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2503\" y=\"-71.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.1464</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2543,-57.5 2543,-93.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2584\" y=\"-71.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.8284</text>\n",
       "</g>\n",
       "<!-- 140404045670992&#45;&gt;140404045524432* -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140404045670992&#45;&gt;140404045524432*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2625.18,-89.15C2634.42,-90.54 2643.38,-91.9 2651.47,-93.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2651.15,-96.61 2661.56,-94.65 2652.19,-89.69 2651.15,-96.61\"/>\n",
       "</g>\n",
       "<!-- 140404045670992**&#45;1 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404045670992**&#45;1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2316\" cy=\"-71.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2316\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">**&#45;1</text>\n",
       "</g>\n",
       "<!-- 140404045670992**&#45;1&#45;&gt;140404045670992 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404045670992**&#45;1&#45;&gt;140404045670992</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2343.05,-71.98C2365.53,-72.4 2399.48,-73.03 2432.5,-73.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2432.64,-77.14 2442.7,-73.83 2432.77,-70.14 2432.64,-77.14\"/>\n",
       "</g>\n",
       "<!-- 140404045669520 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404045669520</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-54.5 1071,-90.5 1256,-90.5 1256,-54.5 1071,-54.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-54.5 1094,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-54.5 1174,-90.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045669520&#45;&gt;140404045669904* -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140404045669520&#45;&gt;140404045669904*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-88.6C1265.49,-90.26 1274.66,-91.88 1282.9,-93.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1282.36,-96.78 1292.81,-95.07 1283.57,-89.89 1282.36,-96.78\"/>\n",
       "</g>\n",
       "<!-- 140404045669520+ -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404045669520+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-72.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045669520+&#45;&gt;140404045669520 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404045669520+&#45;&gt;140404045669520</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-72.5C1042.58,-72.5 1051.3,-72.5 1060.57,-72.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-76 1070.81,-72.5 1060.81,-69 1060.81,-76\"/>\n",
       "</g>\n",
       "<!-- 140404045668496 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404045668496</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-110.5 326,-146.5 546,-146.5 546,-110.5 326,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-110.5 379,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-110.5 464,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045669264+ -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140404045669264+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045668496&#45;&gt;140404045669264+ -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404045668496&#45;&gt;140404045669264+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-110.61C555.64,-109.07 564.6,-107.61 572.65,-106.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.35,-109.72 582.65,-104.65 572.22,-102.81 573.35,-109.72\"/>\n",
       "</g>\n",
       "<!-- 140404045668496*&#45;&gt;140404045668496 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404045668496*&#45;&gt;140404045668496</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C297.77,-128.5 306.37,-128.5 315.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-132 325.84,-128.5 315.84,-125 315.84,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045669008 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404045669008</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-55.5 328.5,-91.5 543.5,-91.5 543.5,-55.5 328.5,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-55.5 381.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-55.5 461.5,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045669008&#45;&gt;140404045669264+ -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140404045669008&#45;&gt;140404045669264+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-90.37C554.01,-91.98 563.76,-93.51 572.44,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.05,-98.37 582.47,-96.47 573.14,-91.45 572.05,-98.37\"/>\n",
       "</g>\n",
       "<!-- 140404045669008*&#45;&gt;140404045669008 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404045669008*&#45;&gt;140404045669008</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C298.51,-73.5 308.08,-73.5 318.36,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-77 328.39,-73.5 318.39,-70 318.39,-77\"/>\n",
       "</g>\n",
       "<!-- 140404045670096 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140404045670096</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1692.5,-81.5 1692.5,-117.5 1874.5,-117.5 1874.5,-81.5 1692.5,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1702.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1712.5,-81.5 1712.5,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1752.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 5.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1792.5,-81.5 1792.5,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1833.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0429</text>\n",
       "</g>\n",
       "<!-- 140404045670096&#45;&gt;140404045670416+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404045670096&#45;&gt;140404045670416+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1874.82,-115.28C1885.07,-117.08 1895.01,-118.81 1903.88,-120.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1903.3,-123.81 1913.75,-122.09 1904.5,-116.92 1903.3,-123.81\"/>\n",
       "</g>\n",
       "<!-- 140404045670736+ -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140404045670736+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1940\" cy=\"-71.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1940\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045670096&#45;&gt;140404045670736+ -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404045670096&#45;&gt;140404045670736+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1874.82,-83.13C1885.07,-81.27 1895.01,-79.47 1903.88,-77.87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1904.54,-81.3 1913.75,-76.08 1903.29,-74.42 1904.54,-81.3\"/>\n",
       "</g>\n",
       "<!-- 140404045670096exp&#45;&gt;140404045670096 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404045670096exp&#45;&gt;140404045670096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1654.21,-99.5C1662.34,-99.5 1671.85,-99.5 1681.94,-99.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1682.19,-103 1692.19,-99.5 1682.19,-96 1682.19,-103\"/>\n",
       "</g>\n",
       "<!-- 140404045669264 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140404045669264</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045669264&#45;&gt;140404045669520+ -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140404045669264&#45;&gt;140404045669520+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M936.53,-82.49C949.09,-80.71 960.99,-79.02 971.3,-77.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-80.99 981.44,-76.12 971.05,-74.06 972.03,-80.99\"/>\n",
       "</g>\n",
       "<!-- 140404045669264+&#45;&gt;140404045669264 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404045669264+&#45;&gt;140404045669264</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140404045664208 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140404045664208</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 200,-146.5 200,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-110.5 33,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-110.5 118,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045664208&#45;&gt;140404045668496* -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140404045664208&#45;&gt;140404045668496*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-128.5C209.2,-128.5 217.86,-128.5 225.7,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-132 235.85,-128.5 225.85,-125 225.85,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045668624 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140404045668624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-27.5 716,-63.5 901,-63.5 901,-27.5 716,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-27.5 739,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-27.5 819,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140404045668624&#45;&gt;140404045669520+ -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140404045668624&#45;&gt;140404045669520+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-58.01C926,-61.43 951.59,-64.93 971.37,-67.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.05,-71.12 981.43,-69.01 972,-64.18 971.05,-71.12\"/>\n",
       "</g>\n",
       "<!-- 140404045670672 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140404045670672</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1690.5,-26.5 1690.5,-62.5 1876.5,-62.5 1876.5,-26.5 1690.5,-26.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1700.5\" y=\"-40.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1710.5,-26.5 1710.5,-62.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1750.5\" y=\"-40.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1790.5,-26.5 1790.5,-62.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1833.5\" y=\"-40.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n",
       "</g>\n",
       "<!-- 140404045670672&#45;&gt;140404045670736+ -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140404045670672&#45;&gt;140404045670736+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1876.6,-60.6C1886.15,-62.26 1895.37,-63.88 1903.68,-65.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1903.2,-68.8 1913.65,-67.07 1904.4,-61.9 1903.2,-68.8\"/>\n",
       "</g>\n",
       "<!-- 140404045664080 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140404045664080</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 197.5,-36.5 197.5,-0.5 2.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-0.5 35.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-0.5 115.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140404045664080&#45;&gt;140404045669008* -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140404045664080&#45;&gt;140404045669008*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140404045670736 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140404045670736</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2003,-53.5 2003,-89.5 2189,-89.5 2189,-53.5 2003,-53.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2013\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2023,-53.5 2023,-89.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2063\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2103,-53.5 2103,-89.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2146\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n",
       "</g>\n",
       "<!-- 140404045670736&#45;&gt;140404045670992**&#45;1 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140404045670736&#45;&gt;140404045670992**&#45;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2189.14,-71.5C2220.72,-71.5 2254.24,-71.5 2278.74,-71.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2278.98,-75 2288.98,-71.5 2278.98,-68 2278.98,-75\"/>\n",
       "</g>\n",
       "<!-- 140404045670736+&#45;&gt;140404045670736 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404045670736+&#45;&gt;140404045670736</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1967.12,-71.5C1974.62,-71.5 1983.29,-71.5 1992.5,-71.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1992.69,-75 2002.69,-71.5 1992.69,-68 1992.69,-75\"/>\n",
       "</g>\n",
       "<!-- 140404045670288 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140404045670288</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1690,-136.5 1690,-172.5 1877,-172.5 1877,-136.5 1690,-136.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1700\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1710,-136.5 1710,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1752.5\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1795,-136.5 1795,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1836\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n",
       "</g>\n",
       "<!-- 140404045670288&#45;&gt;140404045670416+ -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140404045670288&#45;&gt;140404045670416+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1877.05,-137.73C1886.54,-136.01 1895.71,-134.35 1903.95,-132.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1904.64,-136.28 1913.85,-131.06 1903.39,-129.4 1904.64,-136.28\"/>\n",
       "</g>\n",
       "<!-- 140404045524432 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140404045524432</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2751,-80.5 2751,-116.5 2936,-116.5 2936,-80.5 2751,-80.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2762.5\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2774,-80.5 2774,-116.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2814\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2854,-80.5 2854,-116.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2895\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404045524432*&#45;&gt;140404045524432 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045524432*&#45;&gt;140404045524432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2715.04,-98.5C2722.58,-98.5 2731.3,-98.5 2740.57,-98.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2740.81,-102 2750.81,-98.5 2740.81,-95 2740.81,-102\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "\n",
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n'\n",
    "# -----\n",
    "e = (2*n).exp()\n",
    "o = (e - 1)/(e + 1)\n",
    "# -----\n",
    "o.label = 'o'\n",
    "o.backward()\n",
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `x1w1 + x2w2 + b` with PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7071066904050358</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0.7071066904050358\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">-----\n",
       "</pre>\n"
      ],
      "text/plain": [
       "-----\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">x2 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5000001283844369</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "x2 \u001b[1;36m0.5000001283844369\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">w2 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "w2 \u001b[1;36m0.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">x1 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.5000003851533106</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "x1 \u001b[1;36m-1.5000003851533106\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">w1 <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0000002567688737</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "w1 \u001b[1;36m1.0000002567688737\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1 = torch.Tensor([2.0]).double(); x1.requires_grad = True\n",
    "x2 = torch.Tensor([0.0]).double(); x2.requires_grad = True\n",
    "w1 = torch.Tensor([-3.0]).double(); w1.requires_grad = True\n",
    "w2 = torch.Tensor([1.0]).double(); w2.requires_grad = True\n",
    "b = torch.Tensor([6.8813735870195432]); b.requires_grad = True\n",
    "n = x1*w1 + x2*w2 + b\n",
    "o = torch.tanh(n)\n",
    "\n",
    "print(o.data.item())\n",
    "o.backward()\n",
    "\n",
    "print('-----')\n",
    "print('x2', x2.grad.item())\n",
    "print('w2', w2.grad.item())\n",
    "print('x1', x1.grad.item())\n",
    "print('w1', w1.grad.item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>.<span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>.<span style=\"font-weight: bold\">]])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m., \u001b[1;36m2\u001b[0m., \u001b[1;36m3\u001b[0m.\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m4\u001b[0m., \u001b[1;36m5\u001b[0m., \u001b[1;36m6\u001b[0m.\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.Tensor([[1, 2, 3], [4, 5, 6]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.71630401051218</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.71630401051218\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class Neuron:\n",
    "    def __init__(self, nin):\n",
    "        self.w = [Value(random.uniform(-1, 1)) for _ in range(nin)]\n",
    "        self.b = Value(random.uniform(-1, 1))\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        act =  sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
    "        return act.tanh()\n",
    "\n",
    "    def parameters(self):\n",
    "        return self.w + [self.b]\n",
    "    \n",
    "x = [2.0, 3.0]\n",
    "n = Neuron(2)\n",
    "n(x) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Layer:\n",
    "    def __init__(self, nin, nout):\n",
    "        self.neurons = [Neuron(nin\n",
    "                              ) for _ in range(nout)]\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        outs = [n(x) for n in self.neurons]\n",
    "        return outs[0] if len(outs) == 1 else outs\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for neuron in self.neurons for p in neuron.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9323923071860208</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.6957480842688355</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9949508713128399</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9323923071860208\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.6957480842688355\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9949508713128399\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = [2.0, 3.0]\n",
    "n = Layer(2, 3)\n",
    "n(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP:\n",
    "    def __init__(self, nin, nouts):\n",
    "        sz = [nin] + nouts\n",
    "        self.layers = [Layer(sz[i], sz[i+1]) for i\n",
    "                       in range(len(nouts))]\n",
    "        \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "o = n(x)\n",
    "o.grad = 1\n",
    "o.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8773320545613115</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.21854271535158198</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.13730892829595565</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5436703421639371</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5007041170945776</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9789830631658898</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8050974151663517</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.11016135996456167</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.22124094253907778</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8692488975746844</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.51512083826767</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.15884614255298235</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.9216734804692623</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6165197184222242</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.33389808347375305</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6716163019747723</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7479127489965471</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6913996844396202</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3719520946883914</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0381466491267759</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8036261340897828</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.14331062776761772</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.9904951973594573</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.23265417282124412</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5441204768729622</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.09037344168895323</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.6263186959547287</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7687145874568115</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8067183837857432</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.6695236110998573</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4936725683149976</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.948783805686829</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.362064305878842</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.71706547232376</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.38398098767491895</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.854407056637168</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.43771644655834585</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8122254391243122</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7849921896499341</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7867428242639574</span><span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9508849142793219</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">41</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m(\u001b[0m\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8773320545613115\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.21854271535158198\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.13730892829595565\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5436703421639371\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5007041170945776\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9789830631658898\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8050974151663517\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.11016135996456167\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.22124094253907778\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8692488975746844\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.51512083826767\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.15884614255298235\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.9216734804692623\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6165197184222242\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.33389808347375305\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6716163019747723\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7479127489965471\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6913996844396202\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.3719520946883914\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.0381466491267759\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8036261340897828\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.14331062776761772\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.9904951973594573\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.23265417282124412\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5441204768729622\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.09037344168895323\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.6263186959547287\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.7687145874568115\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8067183837857432\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.6695236110998573\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.4936725683149976\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.948783805686829\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.362064305878842\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.71706547232376\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.38398098767491895\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.854407056637168\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.43771644655834585\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8122254391243122\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.7849921896499341\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7867428242639574\u001b[0m\u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9508849142793219\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[1;36m41\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n.parameters(), len(n.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"5795pt\" height=\"1021pt\"\n",
       " viewBox=\"0.00 0.00 5795.00 1021.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1017)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-1017 5791,-1017 5791,4 -4,4\"/>\n",
       "<!-- 140404120084560 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140404120084560</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1587,-660.5 1587,-696.5 1774,-696.5 1774,-660.5 1587,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1597\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1607,-660.5 1607,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.3241</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1692,-660.5 1692,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404120084688+ -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140404120084688+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-678.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120084560&#45;&gt;140404120084688+ -->\n",
       "<g id=\"edge155\" class=\"edge\">\n",
       "<title>140404120084560&#45;&gt;140404120084688+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1774.35,-678.5C1783.97,-678.5 1793.29,-678.5 1801.7,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.81,-682 1811.81,-678.5 1801.81,-675 1801.81,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120084560+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140404120084560+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-678.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120084560+&#45;&gt;140404120084560 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140404120084560+&#45;&gt;140404120084560</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-678.5C1557.22,-678.5 1566.59,-678.5 1576.55,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1576.68,-682 1586.68,-678.5 1576.68,-675 1576.68,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120240208 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140404120240208</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-976.5 634,-1012.5 825,-1012.5 825,-976.5 634,-976.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"644\" y=\"-990.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"654,-976.5 654,-1012.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"696.5\" y=\"-990.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-976.5 739,-1012.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"782\" y=\"-990.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0005</text>\n",
       "</g>\n",
       "<!-- 140404120240272* -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140404120240272*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"888\" cy=\"-939.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120240208&#45;&gt;140404120240272* -->\n",
       "<g id=\"edge190\" class=\"edge\">\n",
       "<title>140404120240208&#45;&gt;140404120240272*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M798.25,-976.44C807.31,-973.65 816.43,-970.64 825,-967.5 835.5,-963.65 846.72,-958.82 856.69,-954.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"858.22,-957.4 865.8,-950 855.26,-951.06 858.22,-957.4\"/>\n",
       "</g>\n",
       "<!-- 140404120240272 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140404120240272</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"951,-914.5 951,-950.5 1142,-950.5 1142,-914.5 951,-914.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"961\" y=\"-928.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"971,-914.5 971,-950.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-928.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1373</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1056,-914.5 1056,-950.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-928.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120240528+ -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140404120240528+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-870.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-866.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120240272&#45;&gt;140404120240528+ -->\n",
       "<g id=\"edge122\" class=\"edge\">\n",
       "<title>140404120240272&#45;&gt;140404120240528+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1100.26,-914.35C1113.94,-909.42 1128.59,-903.93 1142,-898.5 1152.37,-894.3 1163.55,-889.38 1173.53,-884.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1175.04,-888 1182.66,-880.64 1172.11,-881.64 1175.04,-888\"/>\n",
       "</g>\n",
       "<!-- 140404120240272*&#45;&gt;140404120240272 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140404120240272*&#45;&gt;140404120240272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M915.19,-938.33C922.67,-937.99 931.31,-937.61 940.51,-937.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"940.84,-940.68 950.68,-936.74 940.53,-933.69 940.84,-940.68\"/>\n",
       "</g>\n",
       "<!-- 140404120256656 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140404120256656</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-385.5 3487,-421.5 3678,-421.5 3678,-385.5 3487,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-385.5 3507,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5460</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-385.5 3592,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0332</text>\n",
       "</g>\n",
       "<!-- 140404120272208* -->\n",
       "<g id=\"node189\" class=\"node\">\n",
       "<title>140404120272208*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3741\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3741\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120256656&#45;&gt;140404120272208* -->\n",
       "<g id=\"edge207\" class=\"edge\">\n",
       "<title>140404120256656&#45;&gt;140404120272208*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3678.15,-403.5C3687.09,-403.5 3695.73,-403.5 3703.57,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3703.73,-407 3713.73,-403.5 3703.73,-400 3703.73,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120256656tanh -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140404120256656tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3424\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3424\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120256656tanh&#45;&gt;140404120256656 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140404120256656tanh&#45;&gt;140404120256656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3451.19,-403.5C3458.67,-403.5 3467.31,-403.5 3476.51,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3476.68,-407 3486.68,-403.5 3476.68,-400 3476.68,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120084688 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140404120084688</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904,-660.5 1904,-696.5 2091,-696.5 2091,-660.5 1904,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924,-660.5 1924,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.8090</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2009,-660.5 2009,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404045524624tanh -->\n",
       "<g id=\"node93\" class=\"node\">\n",
       "<title>140404045524624tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-677.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-673.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120084688&#45;&gt;140404045524624tanh -->\n",
       "<g id=\"edge176\" class=\"edge\">\n",
       "<title>140404120084688&#45;&gt;140404045524624tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2091.35,-677.91C2100.97,-677.85 2110.29,-677.79 2118.7,-677.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.83,-681.23 2128.81,-677.67 2118.78,-674.23 2118.83,-681.23\"/>\n",
       "</g>\n",
       "<!-- 140404120084688+&#45;&gt;140404120084688 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140404120084688+&#45;&gt;140404120084688</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-678.5C1874.22,-678.5 1883.59,-678.5 1893.55,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1893.68,-682 1903.68,-678.5 1893.68,-675 1893.68,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120240528 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140404120240528</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1268,-797.5 1268,-833.5 1459,-833.5 1459,-797.5 1268,-797.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1278\" y=\"-811.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1288,-797.5 1288,-833.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1330.5\" y=\"-811.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.7800</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1373,-797.5 1373,-833.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-811.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120240656tanh -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140404120240656tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-808.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-804.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120240528&#45;&gt;140404120240656tanh -->\n",
       "<g id=\"edge112\" class=\"edge\">\n",
       "<title>140404120240528&#45;&gt;140404120240656tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1459.15,-811.27C1468.18,-810.86 1476.9,-810.47 1484.81,-810.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1485.2,-813.6 1495.04,-809.66 1484.89,-806.61 1485.2,-813.6\"/>\n",
       "</g>\n",
       "<!-- 140404120240528+&#45;&gt;140404120240528 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140404120240528+&#45;&gt;140404120240528</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1229.43,-862.27C1248.48,-855.57 1276.25,-845.81 1301.5,-836.94\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1302.76,-840.21 1311.03,-833.59 1300.43,-833.6 1302.76,-840.21\"/>\n",
       "</g>\n",
       "<!-- 140404120257040 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140404120257040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-825.5 1902,-861.5 2093,-861.5 2093,-825.5 1902,-825.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-825.5 1922,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1354</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-825.5 2007,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120257296+ -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140404120257296+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-843.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120257040&#45;&gt;140404120257296+ -->\n",
       "<g id=\"edge139\" class=\"edge\">\n",
       "<title>140404120257040&#45;&gt;140404120257296+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2093.15,-843.5C2102.09,-843.5 2110.73,-843.5 2118.57,-843.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.73,-847 2128.73,-843.5 2118.73,-840 2118.73,-847\"/>\n",
       "</g>\n",
       "<!-- 140404120257040* -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140404120257040*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-843.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120257040*&#45;&gt;140404120257040 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140404120257040*&#45;&gt;140404120257040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-843.5C1873.67,-843.5 1882.31,-843.5 1891.51,-843.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1891.68,-847 1901.68,-843.5 1891.68,-840 1891.68,-847\"/>\n",
       "</g>\n",
       "<!-- 140404120240656 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140404120240656</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1585,-783.5 1585,-819.5 1776,-819.5 1776,-783.5 1585,-783.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1595\" y=\"-797.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1605,-783.5 1605,-819.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1647.5\" y=\"-797.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9447</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1690,-783.5 1690,-819.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-797.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0353</text>\n",
       "</g>\n",
       "<!-- 140404120240656&#45;&gt;140404120257040* -->\n",
       "<g id=\"edge152\" class=\"edge\">\n",
       "<title>140404120240656&#45;&gt;140404120257040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1748.78,-819.56C1767.67,-824.63 1787.43,-829.93 1803.54,-834.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1802.99,-837.73 1813.56,-836.94 1804.81,-830.97 1802.99,-837.73\"/>\n",
       "</g>\n",
       "<!-- 140404120242384* -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>140404120242384*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-568.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120240656&#45;&gt;140404120242384* -->\n",
       "<g id=\"edge182\" class=\"edge\">\n",
       "<title>140404120240656&#45;&gt;140404120242384*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1703.25,-783.46C1724,-765.4 1755.1,-735.97 1776,-705.5 1800.07,-670.41 1818.62,-624.47 1828.93,-595.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1832.34,-596.55 1832.34,-585.96 1825.74,-594.24 1832.34,-596.55\"/>\n",
       "</g>\n",
       "<!-- 140404120259344* -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>140404120259344*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-938.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-934.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120240656&#45;&gt;140404120259344* -->\n",
       "<g id=\"edge170\" class=\"edge\">\n",
       "<title>140404120240656&#45;&gt;140404120259344*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1762.09,-819.57C1767.02,-822.16 1771.71,-825.12 1776,-828.5 1802.93,-849.74 1820.1,-886.1 1829.38,-911.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1826.15,-912.55 1832.77,-920.82 1832.76,-910.22 1826.15,-912.55\"/>\n",
       "</g>\n",
       "<!-- 140404120269904* -->\n",
       "<g id=\"node131\" class=\"node\">\n",
       "<title>140404120269904*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-376.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-372.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120240656&#45;&gt;140404120269904* -->\n",
       "<g id=\"edge194\" class=\"edge\">\n",
       "<title>140404120240656&#45;&gt;140404120269904*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1706.78,-783.21C1728.8,-765.83 1759.68,-737.61 1776,-705.5 1820.69,-617.57 1792.64,-583.22 1812,-486.5 1817.65,-458.26 1825.37,-426.25 1830.92,-404.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1834.33,-404.88 1833.39,-394.33 1827.54,-403.16 1834.33,-404.88\"/>\n",
       "</g>\n",
       "<!-- 140404120240656tanh&#45;&gt;140404120240656 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140404120240656tanh&#45;&gt;140404120240656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-807.33C1556.67,-806.99 1565.31,-806.61 1574.51,-806.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1574.84,-809.68 1584.68,-805.74 1574.53,-802.69 1574.84,-809.68\"/>\n",
       "</g>\n",
       "<!-- 140404120240720 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140404120240720</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1272.5,-110.5 1272.5,-146.5 1454.5,-146.5 1454.5,-110.5 1272.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1292.5,-110.5 1292.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1372.5,-110.5 1372.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0042</text>\n",
       "</g>\n",
       "<!-- 140404120240784* -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140404120240784*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120240720&#45;&gt;140404120240784* -->\n",
       "<g id=\"edge94\" class=\"edge\">\n",
       "<title>140404120240720&#45;&gt;140404120240784*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1454.63,-128.5C1465.18,-128.5 1475.43,-128.5 1484.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1484.82,-132 1494.82,-128.5 1484.82,-125 1484.82,-132\"/>\n",
       "</g>\n",
       "<!-- 140404120085136 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140404120085136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1270,-660.5 1270,-696.5 1457,-696.5 1457,-660.5 1270,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1280\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1290,-660.5 1290,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.6077</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1375,-660.5 1375,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404120085136&#45;&gt;140404120084560+ -->\n",
       "<g id=\"edge158\" class=\"edge\">\n",
       "<title>140404120085136&#45;&gt;140404120084560+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1457.35,-678.5C1466.97,-678.5 1476.29,-678.5 1484.7,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1484.81,-682 1494.81,-678.5 1484.81,-675 1484.81,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120085136* -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140404120085136*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-678.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120085136*&#45;&gt;140404120085136 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140404120085136*&#45;&gt;140404120085136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.19,-678.5C1240.22,-678.5 1249.59,-678.5 1259.55,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1259.68,-682 1269.68,-678.5 1259.68,-675 1259.68,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120240784 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140404120240784</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1589.5,-110.5 1589.5,-146.5 1771.5,-146.5 1771.5,-110.5 1589.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1599.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1609.5,-110.5 1609.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.8496</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1689.5,-110.5 1689.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120241296+ -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>140404120241296+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120240784&#45;&gt;140404120241296+ -->\n",
       "<g id=\"edge137\" class=\"edge\">\n",
       "<title>140404120240784&#45;&gt;140404120241296+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1771.63,-128.5C1782.18,-128.5 1792.43,-128.5 1801.6,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.82,-132 1811.82,-128.5 1801.82,-125 1801.82,-132\"/>\n",
       "</g>\n",
       "<!-- 140404120240784*&#45;&gt;140404120240784 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140404120240784*&#45;&gt;140404120240784</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-128.5C1557.92,-128.5 1568.23,-128.5 1579.17,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1579.36,-132 1589.36,-128.5 1579.36,-125 1579.36,-132\"/>\n",
       "</g>\n",
       "<!-- 140404045660880 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140404045660880</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"636,-605.5 636,-641.5 823,-641.5 823,-605.5 636,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"646\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"656,-605.5 656,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0014</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"741,-605.5 741,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"782\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120148176+ -->\n",
       "<g id=\"node161\" class=\"node\">\n",
       "<title>140404120148176+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"888\" cy=\"-568.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404045660880&#45;&gt;140404120148176+ -->\n",
       "<g id=\"edge161\" class=\"edge\">\n",
       "<title>140404045660880&#45;&gt;140404120148176+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M795.15,-605.49C805.21,-602.36 815.43,-598.99 825,-595.5 835.26,-591.76 846.25,-587.18 856.08,-582.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"857.76,-585.94 865.46,-578.66 854.9,-579.55 857.76,-585.94\"/>\n",
       "</g>\n",
       "<!-- 140404045660880* -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140404045660880*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045660880*&#45;&gt;140404045660880 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140404045660880*&#45;&gt;140404045660880</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.19,-623.5C606.22,-623.5 615.59,-623.5 625.55,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.68,-627 635.68,-623.5 625.68,-620 625.68,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120257296 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140404120257296</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2219,-825.5 2219,-861.5 2410,-861.5 2410,-825.5 2219,-825.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2229\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2239,-825.5 2239,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.0450</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2324,-825.5 2324,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120257808+ -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>140404120257808+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-733.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120257296&#45;&gt;140404120257808+ -->\n",
       "<g id=\"edge178\" class=\"edge\">\n",
       "<title>140404120257296&#45;&gt;140404120257808+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2393.46,-825.34C2399.31,-822.51 2404.9,-819.26 2410,-815.5 2433.21,-798.38 2428.13,-784.14 2446,-761.5 2447.4,-759.72 2448.89,-757.92 2450.42,-756.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2453.18,-758.3 2457.24,-748.51 2447.96,-753.64 2453.18,-758.3\"/>\n",
       "</g>\n",
       "<!-- 140404120257296+&#45;&gt;140404120257296 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140404120257296+&#45;&gt;140404120257296</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-843.5C2190.67,-843.5 2199.31,-843.5 2208.51,-843.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2208.68,-847 2218.68,-843.5 2208.68,-840 2208.68,-847\"/>\n",
       "</g>\n",
       "<!-- 140404120241040 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140404120241040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"319.5,-633.5 319.5,-669.5 505.5,-669.5 505.5,-633.5 319.5,-633.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-647.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339.5,-633.5 339.5,-669.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"379.5\" y=\"-647.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"419.5,-633.5 419.5,-669.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"462.5\" y=\"-647.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0005</text>\n",
       "</g>\n",
       "<!-- 140404120241040&#45;&gt;140404045660880* -->\n",
       "<g id=\"edge91\" class=\"edge\">\n",
       "<title>140404120241040&#45;&gt;140404045660880*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M505.89,-634.97C516.17,-633.13 526.11,-631.35 534.97,-629.77\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"535.6,-633.21 544.83,-628 534.37,-626.32 535.6,-633.21\"/>\n",
       "</g>\n",
       "<!-- 140404120151056 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140404120151056</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"955.5,-770.5 955.5,-806.5 1137.5,-806.5 1137.5,-770.5 955.5,-770.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"965.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"975.5,-770.5 975.5,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.4425</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1055.5,-770.5 1055.5,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1096.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404045178448+ -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>140404045178448+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-733.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120151056&#45;&gt;140404045178448+ -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>140404120151056&#45;&gt;140404045178448+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1112.15,-770.49C1122.21,-767.36 1132.43,-763.99 1142,-760.5 1152.26,-756.76 1163.25,-752.18 1173.08,-747.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1174.76,-750.94 1182.46,-743.66 1171.9,-744.55 1174.76,-750.94\"/>\n",
       "</g>\n",
       "<!-- 140404120151056* -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140404120151056*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"888\" cy=\"-774.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120151056*&#45;&gt;140404120151056 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140404120151056*&#45;&gt;140404120151056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M914.84,-776.81C923.62,-777.6 934.02,-778.53 945.06,-779.52\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"945.08,-783.03 955.36,-780.44 945.71,-776.06 945.08,-783.03\"/>\n",
       "</g>\n",
       "<!-- 140404120257552 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140404120257552</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2219,-715.5 2219,-751.5 2410,-751.5 2410,-715.5 2219,-715.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2229\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2239,-715.5 2239,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7627</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2324,-715.5 2324,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120257552&#45;&gt;140404120257808+ -->\n",
       "<g id=\"edge156\" class=\"edge\">\n",
       "<title>140404120257552&#45;&gt;140404120257808+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2410.15,-733.5C2419.09,-733.5 2427.73,-733.5 2435.57,-733.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.73,-737 2445.73,-733.5 2435.73,-730 2435.73,-737\"/>\n",
       "</g>\n",
       "<!-- 140404120257552* -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>140404120257552*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120257552*&#45;&gt;140404120257552 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140404120257552*&#45;&gt;140404120257552</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2172.1,-638.18C2175.81,-642.06 2179.67,-646.33 2183,-650.5 2201.21,-673.34 2195.56,-688.06 2219,-705.5 2221.38,-707.27 2223.87,-708.93 2226.45,-710.49\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2225.04,-713.7 2235.5,-715.38 2228.37,-707.55 2225.04,-713.7\"/>\n",
       "</g>\n",
       "<!-- 140404120241296 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>140404120241296</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1906.5,-110.5 1906.5,-146.5 2088.5,-146.5 2088.5,-110.5 1906.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1916.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1926.5,-110.5 1926.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6778</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2006.5,-110.5 2006.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120241872+ -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>140404120241872+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-183.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120241296&#45;&gt;140404120241872+ -->\n",
       "<g id=\"edge191\" class=\"edge\">\n",
       "<title>140404120241296&#45;&gt;140404120241872+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2066.25,-146.56C2075.31,-149.35 2084.43,-152.36 2093,-155.5 2103.5,-159.35 2114.72,-164.18 2124.69,-168.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2123.26,-171.94 2133.8,-173 2126.22,-165.6 2123.26,-171.94\"/>\n",
       "</g>\n",
       "<!-- 140404120241296+&#45;&gt;140404120241296 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140404120241296+&#45;&gt;140404120241296</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-128.5C1874.92,-128.5 1885.23,-128.5 1896.17,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1896.36,-132 1906.36,-128.5 1896.36,-125 1896.36,-132\"/>\n",
       "</g>\n",
       "<!-- 140404120257808 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>140404120257808</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-715.5 2536,-751.5 2727,-751.5 2727,-715.5 2536,-715.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-715.5 2556,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8077</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-715.5 2641,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120258320+ -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>140404120258320+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-678.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120257808&#45;&gt;140404120258320+ -->\n",
       "<g id=\"edge143\" class=\"edge\">\n",
       "<title>140404120257808&#45;&gt;140404120258320+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2697.15,-715.49C2707.21,-712.36 2717.43,-708.99 2727,-705.5 2737.26,-701.76 2748.25,-697.18 2758.08,-692.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2759.76,-695.94 2767.46,-688.66 2756.9,-689.55 2759.76,-695.94\"/>\n",
       "</g>\n",
       "<!-- 140404120257808+&#45;&gt;140404120257808 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140404120257808+&#45;&gt;140404120257808</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-733.5C2507.67,-733.5 2516.31,-733.5 2525.51,-733.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2525.68,-737 2535.68,-733.5 2525.68,-730 2525.68,-737\"/>\n",
       "</g>\n",
       "<!-- 140404120241552 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>140404120241552</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1587,-220.5 1587,-256.5 1774,-256.5 1774,-220.5 1587,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1597\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1607,-220.5 1607,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1692,-220.5 1692,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0023</text>\n",
       "</g>\n",
       "<!-- 140404120241616* -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>140404120241616*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-183.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120241552&#45;&gt;140404120241616* -->\n",
       "<g id=\"edge133\" class=\"edge\">\n",
       "<title>140404120241552&#45;&gt;140404120241616*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1746.15,-220.49C1756.21,-217.36 1766.43,-213.99 1776,-210.5 1786.26,-206.76 1797.25,-202.18 1807.08,-197.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1808.76,-200.94 1816.46,-193.66 1805.9,-194.55 1808.76,-200.94\"/>\n",
       "</g>\n",
       "<!-- 140404120241616 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>140404120241616</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904,-165.5 1904,-201.5 2091,-201.5 2091,-165.5 1904,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924,-165.5 1924,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.3339</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2009,-165.5 2009,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120241616&#45;&gt;140404120241872+ -->\n",
       "<g id=\"edge150\" class=\"edge\">\n",
       "<title>140404120241616&#45;&gt;140404120241872+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2091.35,-183.5C2100.97,-183.5 2110.29,-183.5 2118.7,-183.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.81,-187 2128.81,-183.5 2118.81,-180 2118.81,-187\"/>\n",
       "</g>\n",
       "<!-- 140404120241616*&#45;&gt;140404120241616 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140404120241616*&#45;&gt;140404120241616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-183.5C1874.22,-183.5 1883.59,-183.5 1893.55,-183.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1893.68,-187 1903.68,-183.5 1893.68,-180 1893.68,-187\"/>\n",
       "</g>\n",
       "<!-- 140404120258064 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>140404120258064</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-660.5 2536,-696.5 2727,-696.5 2727,-660.5 2536,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-660.5 2556,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.2205</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-660.5 2641,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120258064&#45;&gt;140404120258320+ -->\n",
       "<g id=\"edge100\" class=\"edge\">\n",
       "<title>140404120258064&#45;&gt;140404120258320+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2727.15,-678.5C2736.09,-678.5 2744.73,-678.5 2752.57,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.73,-682 2762.73,-678.5 2752.73,-675 2752.73,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120258064* -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>140404120258064*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-678.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120258064*&#45;&gt;140404120258064 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140404120258064*&#45;&gt;140404120258064</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-678.5C2507.67,-678.5 2516.31,-678.5 2525.51,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2525.68,-682 2535.68,-678.5 2525.68,-675 2525.68,-682\"/>\n",
       "</g>\n",
       "<!-- 140404045178448 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>140404045178448</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1272.5,-715.5 1272.5,-751.5 1454.5,-751.5 1454.5,-715.5 1272.5,-715.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1292.5,-715.5 1292.5,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.2836</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1372.5,-715.5 1372.5,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404045178448&#45;&gt;140404120084560+ -->\n",
       "<g id=\"edge193\" class=\"edge\">\n",
       "<title>140404045178448&#45;&gt;140404120084560+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1429.15,-715.49C1439.21,-712.36 1449.43,-708.99 1459,-705.5 1469.26,-701.76 1480.25,-697.18 1490.08,-692.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1491.76,-695.94 1499.46,-688.66 1488.9,-689.55 1491.76,-695.94\"/>\n",
       "</g>\n",
       "<!-- 140404045178448+&#45;&gt;140404045178448 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140404045178448+&#45;&gt;140404045178448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.19,-733.5C1240.92,-733.5 1251.23,-733.5 1262.17,-733.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1262.36,-737 1272.36,-733.5 1262.36,-730 1262.36,-737\"/>\n",
       "</g>\n",
       "<!-- 140404120151760 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>140404120151760</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1906.5,-495.5 1906.5,-531.5 2088.5,-531.5 2088.5,-495.5 1906.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1916.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1926.5,-495.5 1926.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7700</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2006.5,-495.5 2006.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0023</text>\n",
       "</g>\n",
       "<!-- 140404120151760&#45;&gt;140404120257552* -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>140404120151760&#45;&gt;140404120257552*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2077.19,-531.54C2082.75,-534.14 2088.08,-537.11 2093,-540.5 2114.15,-555.09 2131.01,-579.24 2141.77,-597.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2138.76,-599.54 2146.7,-606.56 2144.87,-596.13 2138.76,-599.54\"/>\n",
       "</g>\n",
       "<!-- 140404120242896* -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>140404120242896*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120151760&#45;&gt;140404120242896* -->\n",
       "<g id=\"edge135\" class=\"edge\">\n",
       "<title>140404120151760&#45;&gt;140404120242896*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2076.46,-495.34C2082.31,-492.51 2087.9,-489.26 2093,-485.5 2116.21,-468.38 2111.13,-454.14 2129,-431.5 2130.4,-429.72 2131.89,-427.92 2133.42,-426.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2136.18,-428.3 2140.24,-418.51 2130.96,-423.64 2136.18,-428.3\"/>\n",
       "</g>\n",
       "<!-- 140404120259856* -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>140404120259856*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-733.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120151760&#45;&gt;140404120259856* -->\n",
       "<g id=\"edge130\" class=\"edge\">\n",
       "<title>140404120151760&#45;&gt;140404120259856*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2082.5,-531.6C2086.3,-534.19 2089.85,-537.14 2093,-540.5 2144.1,-594.88 2094.97,-638.09 2129,-704.5 2130.16,-706.76 2131.54,-708.99 2133.05,-711.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2130.35,-713.36 2139.34,-718.97 2135.81,-708.98 2130.35,-713.36\"/>\n",
       "</g>\n",
       "<!-- 140404120270416* -->\n",
       "<g id=\"node146\" class=\"node\">\n",
       "<title>140404120270416*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-348.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120151760&#45;&gt;140404120270416* -->\n",
       "<g id=\"edge148\" class=\"edge\">\n",
       "<title>140404120151760&#45;&gt;140404120270416*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2079.91,-495.46C2084.64,-492.61 2089.07,-489.31 2093,-485.5 2129.63,-449.99 2103.05,-420.42 2129,-376.5 2130.15,-374.55 2131.46,-372.62 2132.86,-370.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2135.66,-372.85 2139.4,-362.94 2130.3,-368.36 2135.66,-372.85\"/>\n",
       "</g>\n",
       "<!-- 140404120151760tanh -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>140404120151760tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1839\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120151760tanh&#45;&gt;140404120151760 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140404120151760tanh&#45;&gt;140404120151760</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-513.5C1874.92,-513.5 1885.23,-513.5 1896.17,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1896.36,-517 1906.36,-513.5 1896.36,-510 1896.36,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120241872 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>140404120241872</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2223.5,-192.5 2223.5,-228.5 2405.5,-228.5 2405.5,-192.5 2223.5,-192.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2233.5\" y=\"-206.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2243.5,-192.5 2243.5,-228.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-206.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.3439</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2323.5,-192.5 2323.5,-228.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-206.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120242000tanh -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>140404120242000tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-251.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-247.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120241872&#45;&gt;140404120242000tanh -->\n",
       "<g id=\"edge87\" class=\"edge\">\n",
       "<title>140404120241872&#45;&gt;140404120242000tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2384.6,-228.6C2403,-233.42 2422.09,-238.43 2437.72,-242.52\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2436.88,-245.92 2447.44,-245.07 2438.66,-239.15 2436.88,-245.92\"/>\n",
       "</g>\n",
       "<!-- 140404120241872+&#45;&gt;140404120241872 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140404120241872+&#45;&gt;140404120241872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2182.49,-187.9C2191.4,-189.44 2202.02,-191.27 2213.3,-193.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2212.91,-196.7 2223.36,-194.95 2214.1,-189.8 2212.91,-196.7\"/>\n",
       "</g>\n",
       "<!-- 140404120258320 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>140404120258320</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2853,-660.5 2853,-696.5 3044,-696.5 3044,-660.5 2853,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2863\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2873,-660.5 2873,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0282</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2958,-660.5 2958,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120258832+ -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>140404120258832+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3107\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3107\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120258320&#45;&gt;140404120258832+ -->\n",
       "<g id=\"edge142\" class=\"edge\">\n",
       "<title>140404120258320&#45;&gt;140404120258832+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3029.55,-660.47C3034.73,-657.62 3039.61,-654.32 3044,-650.5 3061.33,-635.39 3084.16,-576.23 3096.72,-540.78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3100.13,-541.64 3100.12,-531.04 3093.52,-539.33 3100.13,-541.64\"/>\n",
       "</g>\n",
       "<!-- 140404120258320+&#45;&gt;140404120258320 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140404120258320+&#45;&gt;140404120258320</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-678.5C2824.67,-678.5 2833.31,-678.5 2842.51,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2842.68,-682 2852.68,-678.5 2842.68,-675 2842.68,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120242000 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>140404120242000</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2540.5,-275.5 2540.5,-311.5 2722.5,-311.5 2722.5,-275.5 2540.5,-275.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2550.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2560.5,-275.5 2560.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2600.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.3310</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2640.5,-275.5 2640.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2681.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0077</text>\n",
       "</g>\n",
       "<!-- 140404120258576* -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>140404120258576*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120242000&#45;&gt;140404120258576* -->\n",
       "<g id=\"edge154\" class=\"edge\">\n",
       "<title>140404120242000&#45;&gt;140404120258576*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2715.69,-311.69C2719.75,-314.26 2723.57,-317.18 2727,-320.5 2750.92,-343.65 2773.35,-437.99 2783.42,-485.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2780.03,-486.54 2785.49,-495.62 2786.89,-485.12 2780.03,-486.54\"/>\n",
       "</g>\n",
       "<!-- 140404120243920* -->\n",
       "<g id=\"node96\" class=\"node\">\n",
       "<title>140404120243920*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-293.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120242000&#45;&gt;140404120243920* -->\n",
       "<g id=\"edge123\" class=\"edge\">\n",
       "<title>140404120242000&#45;&gt;140404120243920*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2722.63,-293.5C2733.18,-293.5 2743.43,-293.5 2752.6,-293.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.82,-297 2762.82,-293.5 2752.82,-290 2752.82,-297\"/>\n",
       "</g>\n",
       "<!-- 140404120269136* -->\n",
       "<g id=\"node110\" class=\"node\">\n",
       "<title>140404120269136*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-568.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120242000&#45;&gt;140404120269136* -->\n",
       "<g id=\"edge144\" class=\"edge\">\n",
       "<title>140404120242000&#45;&gt;140404120269136*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2717.13,-311.7C2720.73,-314.27 2724.05,-317.19 2727,-320.5 2792.88,-394.51 2719.43,-451.51 2763,-540.5 2764,-542.53 2765.19,-544.52 2766.51,-546.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2763.88,-548.76 2772.88,-554.34 2769.33,-544.36 2763.88,-548.76\"/>\n",
       "</g>\n",
       "<!-- 140404120271440* -->\n",
       "<g id=\"node170\" class=\"node\">\n",
       "<title>140404120271440*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-238.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120242000&#45;&gt;140404120271440* -->\n",
       "<g id=\"edge210\" class=\"edge\">\n",
       "<title>140404120242000&#45;&gt;140404120271440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2697.15,-275.49C2707.21,-272.36 2717.43,-268.99 2727,-265.5 2737.26,-261.76 2748.25,-257.18 2758.08,-252.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2759.76,-255.94 2767.46,-248.66 2756.9,-249.55 2759.76,-255.94\"/>\n",
       "</g>\n",
       "<!-- 140404120242000tanh&#45;&gt;140404120242000 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140404120242000tanh&#45;&gt;140404120242000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2498.45,-258.06C2513.45,-262.09 2533.56,-267.48 2553.55,-272.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2552.77,-276.26 2563.34,-275.47 2554.58,-269.5 2552.77,-276.26\"/>\n",
       "</g>\n",
       "<!-- 140404120258576 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>140404120258576</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2853,-495.5 2853,-531.5 3044,-531.5 3044,-495.5 2853,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2863\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2873,-495.5 2873,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1801</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2958,-495.5 2958,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120258576&#45;&gt;140404120258832+ -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>140404120258576&#45;&gt;140404120258832+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3044.15,-513.5C3053.09,-513.5 3061.73,-513.5 3069.57,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3069.73,-517 3079.73,-513.5 3069.73,-510 3069.73,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120258576*&#45;&gt;140404120258576 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140404120258576*&#45;&gt;140404120258576</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-513.5C2824.67,-513.5 2833.31,-513.5 2842.51,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2842.68,-517 2852.68,-513.5 2842.68,-510 2842.68,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120242384 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>140404120242384</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-550.5 1902,-586.5 2093,-586.5 2093,-550.5 1902,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-550.5 1922,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7066</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-550.5 2007,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120242640+ -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>140404120242640+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-458.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120242384&#45;&gt;140404120242640+ -->\n",
       "<g id=\"edge132\" class=\"edge\">\n",
       "<title>140404120242384&#45;&gt;140404120242640+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2075.26,-550.44C2081.51,-547.59 2087.51,-544.3 2093,-540.5 2113.74,-526.14 2130.48,-502.63 2141.31,-484.46\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2144.5,-485.94 2146.43,-475.52 2138.42,-482.46 2144.5,-485.94\"/>\n",
       "</g>\n",
       "<!-- 140404120242384*&#45;&gt;140404120242384 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140404120242384*&#45;&gt;140404120242384</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.19,-568.5C1873.67,-568.5 1882.31,-568.5 1891.51,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1891.68,-572 1901.68,-568.5 1891.68,-565 1891.68,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120258832 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>140404120258832</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3170,-495.5 3170,-531.5 3361,-531.5 3361,-495.5 3170,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3180\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3190,-495.5 3190,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3232.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.2083</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3275,-495.5 3275,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3318\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120258960tanh -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>140404120258960tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3424\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3424\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120258832&#45;&gt;140404120258960tanh -->\n",
       "<g id=\"edge157\" class=\"edge\">\n",
       "<title>140404120258832&#45;&gt;140404120258960tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3361.15,-513.5C3370.09,-513.5 3378.73,-513.5 3386.57,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3386.73,-517 3396.73,-513.5 3386.73,-510 3386.73,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120258832+&#45;&gt;140404120258832 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140404120258832+&#45;&gt;140404120258832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3134.19,-513.5C3141.67,-513.5 3150.31,-513.5 3159.51,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3159.68,-517 3169.68,-513.5 3159.68,-510 3159.68,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120258960 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>140404120258960</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-495.5 3487,-531.5 3678,-531.5 3678,-495.5 3487,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-495.5 3507,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8362</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-495.5 3592,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0616</text>\n",
       "</g>\n",
       "<!-- 140404120272720* -->\n",
       "<g id=\"node198\" class=\"node\">\n",
       "<title>140404120272720*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3741\" cy=\"-458.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3741\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120258960&#45;&gt;140404120272720* -->\n",
       "<g id=\"edge188\" class=\"edge\">\n",
       "<title>140404120258960&#45;&gt;140404120272720*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3648.15,-495.49C3658.21,-492.36 3668.43,-488.99 3678,-485.5 3688.26,-481.76 3699.25,-477.18 3709.08,-472.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3710.76,-475.94 3718.46,-468.66 3707.9,-469.55 3710.76,-475.94\"/>\n",
       "</g>\n",
       "<!-- 140404120258960tanh&#45;&gt;140404120258960 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140404120258960tanh&#45;&gt;140404120258960</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3451.19,-513.5C3458.67,-513.5 3467.31,-513.5 3476.51,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3476.68,-517 3486.68,-513.5 3476.68,-510 3476.68,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120177104 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>140404120177104</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-894.5 0,-930.5 191,-930.5 191,-894.5 0,-894.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"10\" y=\"-908.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"20,-894.5 20,-930.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-908.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8773</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"105,-894.5 105,-930.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-908.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0076</text>\n",
       "</g>\n",
       "<!-- 140404120214480* -->\n",
       "<g id=\"node181\" class=\"node\">\n",
       "<title>140404120214480*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"254\" cy=\"-884.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"254\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120177104&#45;&gt;140404120214480* -->\n",
       "<g id=\"edge168\" class=\"edge\">\n",
       "<title>140404120177104&#45;&gt;140404120214480*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M191.15,-895.57C200.46,-893.9 209.44,-892.29 217.54,-890.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"218.41,-894.24 227.64,-889.04 217.18,-887.35 218.41,-894.24\"/>\n",
       "</g>\n",
       "<!-- 140404120242640 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>140404120242640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2219,-440.5 2219,-476.5 2410,-476.5 2410,-440.5 2219,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2229\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2239,-440.5 2239,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.5102</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2324,-440.5 2324,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120243152+ -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>140404120243152+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120242640&#45;&gt;140404120243152+ -->\n",
       "<g id=\"edge97\" class=\"edge\">\n",
       "<title>140404120242640&#45;&gt;140404120243152+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2380.15,-440.49C2390.21,-437.36 2400.43,-433.99 2410,-430.5 2420.26,-426.76 2431.25,-422.18 2441.08,-417.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2442.76,-420.94 2450.46,-413.66 2439.9,-414.55 2442.76,-420.94\"/>\n",
       "</g>\n",
       "<!-- 140404120242640+&#45;&gt;140404120242640 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140404120242640+&#45;&gt;140404120242640</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-458.5C2190.67,-458.5 2199.31,-458.5 2208.51,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2208.68,-462 2218.68,-458.5 2208.68,-455 2208.68,-462\"/>\n",
       "</g>\n",
       "<!-- 140404120177168 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>140404120177168</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"319.5,-811.5 319.5,-847.5 505.5,-847.5 505.5,-811.5 319.5,-811.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339.5,-811.5 339.5,-847.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"379.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.2185</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"419.5,-811.5 419.5,-847.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"462.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0114</text>\n",
       "</g>\n",
       "<!-- 140404120215056* -->\n",
       "<g id=\"node192\" class=\"node\">\n",
       "<title>140404120215056*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-829.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120177168&#45;&gt;140404120215056* -->\n",
       "<g id=\"edge201\" class=\"edge\">\n",
       "<title>140404120177168&#45;&gt;140404120215056*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M505.89,-829.5C515.67,-829.5 525.13,-829.5 533.66,-829.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"533.91,-833 543.91,-829.5 533.91,-826 533.91,-833\"/>\n",
       "</g>\n",
       "<!-- 140404120152656 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>140404120152656</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1270,-440.5 1270,-476.5 1457,-476.5 1457,-440.5 1270,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1280\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1290,-440.5 1290,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8051</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1375,-440.5 1375,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120153104+ -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>140404120153104+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120152656&#45;&gt;140404120153104+ -->\n",
       "<g id=\"edge187\" class=\"edge\">\n",
       "<title>140404120152656&#45;&gt;140404120153104+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1432.25,-476.56C1441.31,-479.35 1450.43,-482.36 1459,-485.5 1469.5,-489.35 1480.72,-494.18 1490.69,-498.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1489.26,-501.94 1499.8,-503 1492.22,-495.6 1489.26,-501.94\"/>\n",
       "</g>\n",
       "<!-- 140404120152656* -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>140404120152656*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-458.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120152656*&#45;&gt;140404120152656 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140404120152656*&#45;&gt;140404120152656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.19,-458.5C1240.22,-458.5 1249.59,-458.5 1259.55,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1259.68,-462 1269.68,-458.5 1259.68,-455 1259.68,-462\"/>\n",
       "</g>\n",
       "<!-- 140404120242896 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>140404120242896</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221.5,-385.5 2221.5,-421.5 2407.5,-421.5 2407.5,-385.5 2221.5,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241.5,-385.5 2241.5,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.5324</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2321.5,-385.5 2321.5,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120242896&#45;&gt;140404120243152+ -->\n",
       "<g id=\"edge165\" class=\"edge\">\n",
       "<title>140404120242896&#45;&gt;140404120243152+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2407.89,-403.5C2417.67,-403.5 2427.13,-403.5 2435.66,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.91,-407 2445.91,-403.5 2435.91,-400 2435.91,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120242896*&#45;&gt;140404120242896 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>140404120242896*&#45;&gt;140404120242896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-403.5C2191.35,-403.5 2200.89,-403.5 2211.03,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2211.35,-407 2221.35,-403.5 2211.35,-400 2211.35,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120259344 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>140404120259344</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904.5,-935.5 1904.5,-971.5 2090.5,-971.5 2090.5,-935.5 1904.5,-935.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924.5,-935.5 1924.5,-971.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.5917</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2004.5,-935.5 2004.5,-971.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120259600+ -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>140404120259600+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-953.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120259344&#45;&gt;140404120259600+ -->\n",
       "<g id=\"edge90\" class=\"edge\">\n",
       "<title>140404120259344&#45;&gt;140404120259600+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2090.89,-953.5C2100.67,-953.5 2110.13,-953.5 2118.66,-953.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.91,-957 2128.91,-953.5 2118.91,-950 2118.91,-957\"/>\n",
       "</g>\n",
       "<!-- 140404120259344*&#45;&gt;140404120259344 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>140404120259344*&#45;&gt;140404120259344</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1865.84,-940.98C1874.13,-941.77 1883.87,-942.71 1894.24,-943.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1894.05,-947.2 1904.34,-944.67 1894.72,-940.23 1894.05,-947.2\"/>\n",
       "</g>\n",
       "<!-- 140404120087504 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>140404120087504</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953.5,-660.5 953.5,-696.5 1139.5,-696.5 1139.5,-660.5 953.5,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973.5,-660.5 973.5,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1053.5,-660.5 1053.5,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1096.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0011</text>\n",
       "</g>\n",
       "<!-- 140404120087504&#45;&gt;140404120085136* -->\n",
       "<g id=\"edge203\" class=\"edge\">\n",
       "<title>140404120087504&#45;&gt;140404120085136*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1139.89,-678.5C1149.67,-678.5 1159.13,-678.5 1167.66,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.91,-682 1177.91,-678.5 1167.91,-675 1167.91,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120153040 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>140404120153040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1272.5,-495.5 1272.5,-531.5 1454.5,-531.5 1454.5,-495.5 1272.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1292.5,-495.5 1292.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.8254</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1372.5,-495.5 1372.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120153040&#45;&gt;140404120153104+ -->\n",
       "<g id=\"edge205\" class=\"edge\">\n",
       "<title>140404120153040&#45;&gt;140404120153104+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1454.63,-513.5C1465.18,-513.5 1475.43,-513.5 1484.6,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1484.82,-517 1494.82,-513.5 1484.82,-510 1484.82,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120153040+ -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>140404120153040+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120153040+&#45;&gt;140404120153040 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>140404120153040+&#45;&gt;140404120153040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.19,-513.5C1240.92,-513.5 1251.23,-513.5 1262.17,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1262.36,-517 1272.36,-513.5 1262.36,-510 1262.36,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120243152 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>140404120243152</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-385.5 2536,-421.5 2727,-421.5 2727,-385.5 2536,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-385.5 2556,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9778</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-385.5 2641,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120243664+ -->\n",
       "<g id=\"node85\" class=\"node\">\n",
       "<title>140404120243664+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120243152&#45;&gt;140404120243664+ -->\n",
       "<g id=\"edge96\" class=\"edge\">\n",
       "<title>140404120243152&#45;&gt;140404120243664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2716.5,-421.6C2720.31,-424.19 2723.85,-427.14 2727,-430.5 2778.36,-485.23 2728.23,-528.98 2763,-595.5 2764.05,-597.51 2765.28,-599.48 2766.63,-601.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2764.02,-603.72 2773.06,-609.25 2769.44,-599.29 2764.02,-603.72\"/>\n",
       "</g>\n",
       "<!-- 140404120243152+&#45;&gt;140404120243152 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>140404120243152+&#45;&gt;140404120243152</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-403.5C2507.67,-403.5 2516.31,-403.5 2525.51,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2525.68,-407 2535.68,-403.5 2525.68,-400 2525.68,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120153104 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>140404120153104</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1589.5,-495.5 1589.5,-531.5 1771.5,-531.5 1771.5,-495.5 1589.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1599.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1609.5,-495.5 1609.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0203</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1689.5,-495.5 1689.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120153104&#45;&gt;140404120151760tanh -->\n",
       "<g id=\"edge126\" class=\"edge\">\n",
       "<title>140404120153104&#45;&gt;140404120151760tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1771.63,-513.5C1782.18,-513.5 1792.43,-513.5 1801.6,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.82,-517 1811.82,-513.5 1801.82,-510 1801.82,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120153104+&#45;&gt;140404120153104 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>140404120153104+&#45;&gt;140404120153104</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-513.5C1557.92,-513.5 1568.23,-513.5 1579.17,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1579.36,-517 1589.36,-513.5 1579.36,-510 1579.36,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120259600 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>140404120259600</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221.5,-935.5 2221.5,-971.5 2407.5,-971.5 2407.5,-935.5 2221.5,-935.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241.5,-935.5 2241.5,-971.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0980</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2321.5,-935.5 2321.5,-971.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-949.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120260112+ -->\n",
       "<g id=\"node90\" class=\"node\">\n",
       "<title>140404120260112+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-843.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120259600&#45;&gt;140404120260112+ -->\n",
       "<g id=\"edge197\" class=\"edge\">\n",
       "<title>140404120259600&#45;&gt;140404120260112+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2394.19,-935.46C2399.75,-932.86 2405.08,-929.89 2410,-926.5 2431.15,-911.91 2448.01,-887.76 2458.77,-869.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2461.87,-870.87 2463.7,-860.44 2455.76,-867.46 2461.87,-870.87\"/>\n",
       "</g>\n",
       "<!-- 140404120259600+&#45;&gt;140404120259600 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>140404120259600+&#45;&gt;140404120259600</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-953.5C2191.35,-953.5 2200.89,-953.5 2211.03,-953.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2211.35,-957 2221.35,-953.5 2211.35,-950 2211.35,-957\"/>\n",
       "</g>\n",
       "<!-- 140404120243408 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>140404120243408</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2538.5,-605.5 2538.5,-641.5 2724.5,-641.5 2724.5,-605.5 2538.5,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2548.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2558.5,-605.5 2558.5,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.3525</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2638.5,-605.5 2638.5,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2681.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120243408&#45;&gt;140404120243664+ -->\n",
       "<g id=\"edge198\" class=\"edge\">\n",
       "<title>140404120243408&#45;&gt;140404120243664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2724.89,-623.5C2734.67,-623.5 2744.13,-623.5 2752.66,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.91,-627 2762.91,-623.5 2752.91,-620 2752.91,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120243408* -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>140404120243408*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120243408*&#45;&gt;140404120243408 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>140404120243408*&#45;&gt;140404120243408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-623.5C2508.35,-623.5 2517.89,-623.5 2528.03,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2528.35,-627 2538.35,-623.5 2528.35,-620 2528.35,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120259856 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>140404120259856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2219,-770.5 2219,-806.5 2410,-806.5 2410,-770.5 2219,-770.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2229\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2239,-770.5 2239,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5919</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2324,-770.5 2324,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120259856&#45;&gt;140404120260112+ -->\n",
       "<g id=\"edge127\" class=\"edge\">\n",
       "<title>140404120259856&#45;&gt;140404120260112+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2383.25,-806.56C2392.31,-809.35 2401.43,-812.36 2410,-815.5 2420.5,-819.35 2431.72,-824.18 2441.69,-828.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2440.26,-831.94 2450.8,-833 2443.22,-825.6 2440.26,-831.94\"/>\n",
       "</g>\n",
       "<!-- 140404120259856*&#45;&gt;140404120259856 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>140404120259856*&#45;&gt;140404120259856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2178.54,-743.66C2190.35,-749.03 2205.32,-755.52 2219,-760.5 2225.43,-762.84 2232.15,-765.13 2238.92,-767.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2238.26,-770.8 2248.85,-770.49 2240.38,-764.13 2238.26,-770.8\"/>\n",
       "</g>\n",
       "<!-- 140404120178000 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>140404120178000</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-921.5 638.5,-957.5 820.5,-957.5 820.5,-921.5 638.5,-921.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"648.5\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-921.5 658.5,-957.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.1373</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-921.5 738.5,-957.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120178000&#45;&gt;140404120240272* -->\n",
       "<g id=\"edge206\" class=\"edge\">\n",
       "<title>140404120178000&#45;&gt;140404120240272*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.63,-939.5C831.18,-939.5 841.43,-939.5 850.6,-939.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"850.82,-943 860.82,-939.5 850.82,-936 850.82,-943\"/>\n",
       "</g>\n",
       "<!-- 140404120088016 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>140404120088016</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1589.5,-605.5 1589.5,-641.5 1771.5,-641.5 1771.5,-605.5 1589.5,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1599.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1609.5,-605.5 1609.5,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.5151</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1689.5,-605.5 1689.5,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404120088016&#45;&gt;140404120084688+ -->\n",
       "<g id=\"edge140\" class=\"edge\">\n",
       "<title>140404120088016&#45;&gt;140404120084688+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1749.25,-641.56C1758.31,-644.35 1767.43,-647.36 1776,-650.5 1786.5,-654.35 1797.72,-659.18 1807.69,-663.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1806.26,-666.94 1816.8,-668 1809.22,-660.6 1806.26,-666.94\"/>\n",
       "</g>\n",
       "<!-- 140404120088016* -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>140404120088016*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120088016*&#45;&gt;140404120088016 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>140404120088016*&#45;&gt;140404120088016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-623.5C1557.92,-623.5 1568.23,-623.5 1579.17,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1579.36,-627 1589.36,-623.5 1579.36,-620 1579.36,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120178128 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>140404120178128</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-921.5 317,-957.5 508,-957.5 508,-921.5 317,-921.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"337,-921.5 337,-957.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"379.5\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5437</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"422,-921.5 422,-957.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-935.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120214736+ -->\n",
       "<g id=\"node185\" class=\"node\">\n",
       "<title>140404120214736+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"571\" cy=\"-884.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120178128&#45;&gt;140404120214736+ -->\n",
       "<g id=\"edge195\" class=\"edge\">\n",
       "<title>140404120178128&#45;&gt;140404120214736+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M481.25,-921.44C490.31,-918.65 499.43,-915.64 508,-912.5 518.5,-908.65 529.72,-903.82 539.69,-899.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"541.22,-902.4 548.8,-895 538.26,-896.06 541.22,-902.4\"/>\n",
       "</g>\n",
       "<!-- 140404120243664 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>140404120243664</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2853,-605.5 2853,-641.5 3044,-641.5 3044,-605.5 2853,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2863\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2873,-605.5 2873,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6253</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2958,-605.5 2958,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120244176+ -->\n",
       "<g id=\"node102\" class=\"node\">\n",
       "<title>140404120244176+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3107\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3107\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120243664&#45;&gt;140404120244176+ -->\n",
       "<g id=\"edge134\" class=\"edge\">\n",
       "<title>140404120243664&#45;&gt;140404120244176+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3030.94,-605.49C3035.67,-602.63 3040.08,-599.32 3044,-595.5 3045.53,-594.01 3080.7,-483.39 3097.38,-430.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3100.77,-431.64 3100.45,-421.05 3094.09,-429.53 3100.77,-431.64\"/>\n",
       "</g>\n",
       "<!-- 140404120243664+&#45;&gt;140404120243664 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>140404120243664+&#45;&gt;140404120243664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-623.5C2824.67,-623.5 2833.31,-623.5 2842.51,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2842.68,-627 2852.68,-623.5 2842.68,-620 2842.68,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120088080 -->\n",
       "<g id=\"node86\" class=\"node\">\n",
       "<title>140404120088080</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1268,-550.5 1268,-586.5 1459,-586.5 1459,-550.5 1268,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1278\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1288,-550.5 1288,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1330.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1373,-550.5 1373,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0006</text>\n",
       "</g>\n",
       "<!-- 140404120088080&#45;&gt;140404120088016* -->\n",
       "<g id=\"edge93\" class=\"edge\">\n",
       "<title>140404120088080&#45;&gt;140404120088016*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1432.25,-586.56C1441.31,-589.35 1450.43,-592.36 1459,-595.5 1469.5,-599.35 1480.72,-604.18 1490.69,-608.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1489.26,-611.94 1499.8,-613 1492.22,-605.6 1489.26,-611.94\"/>\n",
       "</g>\n",
       "<!-- 140404120153616 -->\n",
       "<g id=\"node87\" class=\"node\">\n",
       "<title>140404120153616</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-756.5 638.5,-792.5 820.5,-792.5 820.5,-756.5 638.5,-756.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"648.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-756.5 658.5,-792.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-756.5 738.5,-792.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0003</text>\n",
       "</g>\n",
       "<!-- 140404120153616&#45;&gt;140404120151056* -->\n",
       "<g id=\"edge106\" class=\"edge\">\n",
       "<title>140404120153616&#45;&gt;140404120151056*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.63,-774.5C831.18,-774.5 841.43,-774.5 850.6,-774.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"850.82,-778 860.82,-774.5 850.82,-771 850.82,-778\"/>\n",
       "</g>\n",
       "<!-- 140404120178192 -->\n",
       "<g id=\"node88\" class=\"node\">\n",
       "<title>140404120178192</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"319,-578.5 319,-614.5 506,-614.5 506,-578.5 319,-578.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-592.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339,-578.5 339,-614.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"381.5\" y=\"-592.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5007</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"424,-578.5 424,-614.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-592.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0019</text>\n",
       "</g>\n",
       "<!-- 140404120178192&#45;&gt;140404045660880* -->\n",
       "<g id=\"edge98\" class=\"edge\">\n",
       "<title>140404120178192&#45;&gt;140404045660880*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M506.35,-612.52C516.36,-614.25 526.04,-615.92 534.71,-617.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"534.28,-620.89 544.73,-619.14 535.47,-613.99 534.28,-620.89\"/>\n",
       "</g>\n",
       "<!-- 140404120260112 -->\n",
       "<g id=\"node89\" class=\"node\">\n",
       "<title>140404120260112</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-825.5 2536,-861.5 2727,-861.5 2727,-825.5 2536,-825.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-825.5 2556,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.4939</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-825.5 2641,-861.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-839.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120268880+ -->\n",
       "<g id=\"node104\" class=\"node\">\n",
       "<title>140404120268880+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-788.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120260112&#45;&gt;140404120268880+ -->\n",
       "<g id=\"edge119\" class=\"edge\">\n",
       "<title>140404120260112&#45;&gt;140404120268880+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2697.15,-825.49C2707.21,-822.36 2717.43,-818.99 2727,-815.5 2737.26,-811.76 2748.25,-807.18 2758.08,-802.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2759.76,-805.94 2767.46,-798.66 2756.9,-799.55 2759.76,-805.94\"/>\n",
       "</g>\n",
       "<!-- 140404120260112+&#45;&gt;140404120260112 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>140404120260112+&#45;&gt;140404120260112</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-843.5C2507.67,-843.5 2516.31,-843.5 2525.51,-843.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2525.68,-847 2535.68,-843.5 2525.68,-840 2525.68,-847\"/>\n",
       "</g>\n",
       "<!-- 140404120178256 -->\n",
       "<g id=\"node91\" class=\"node\">\n",
       "<title>140404120178256</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953.5,-440.5 953.5,-476.5 1139.5,-476.5 1139.5,-440.5 953.5,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973.5,-440.5 973.5,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8051</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1053.5,-440.5 1053.5,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1096.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120178256&#45;&gt;140404120152656* -->\n",
       "<g id=\"edge162\" class=\"edge\">\n",
       "<title>140404120178256&#45;&gt;140404120152656*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1139.89,-458.5C1149.67,-458.5 1159.13,-458.5 1167.66,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.91,-462 1177.91,-458.5 1167.91,-455 1167.91,-462\"/>\n",
       "</g>\n",
       "<!-- 140404045524624 -->\n",
       "<g id=\"node92\" class=\"node\">\n",
       "<title>140404045524624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221,-605.5 2221,-641.5 2408,-641.5 2408,-605.5 2221,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241,-605.5 2241,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9477</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2326,-605.5 2326,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0122</text>\n",
       "</g>\n",
       "<!-- 140404045524624&#45;&gt;140404120258064* -->\n",
       "<g id=\"edge192\" class=\"edge\">\n",
       "<title>140404045524624&#45;&gt;140404120258064*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2383.25,-641.56C2392.31,-644.35 2401.43,-647.36 2410,-650.5 2420.5,-654.35 2431.72,-659.18 2441.69,-663.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2440.26,-666.94 2450.8,-668 2443.22,-660.6 2440.26,-666.94\"/>\n",
       "</g>\n",
       "<!-- 140404045524624&#45;&gt;140404120243408* -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>140404045524624&#45;&gt;140404120243408*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2408.35,-623.5C2417.97,-623.5 2427.29,-623.5 2435.7,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.81,-627 2445.81,-623.5 2435.81,-620 2435.81,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120260368* -->\n",
       "<g id=\"node98\" class=\"node\">\n",
       "<title>140404120260368*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-788.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045524624&#45;&gt;140404120260368* -->\n",
       "<g id=\"edge166\" class=\"edge\">\n",
       "<title>140404045524624&#45;&gt;140404120260368*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.67,-641.72C2402.74,-644.27 2406.56,-647.19 2410,-650.5 2447.06,-686.18 2419.89,-716.18 2446,-760.5 2447.15,-762.45 2448.45,-764.38 2449.85,-766.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2447.29,-768.65 2456.39,-774.07 2452.66,-764.15 2447.29,-768.65\"/>\n",
       "</g>\n",
       "<!-- 140404120270928* -->\n",
       "<g id=\"node158\" class=\"node\">\n",
       "<title>140404120270928*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404045524624&#45;&gt;140404120270928* -->\n",
       "<g id=\"edge177\" class=\"edge\">\n",
       "<title>140404045524624&#45;&gt;140404120270928*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2392.26,-605.44C2398.51,-602.59 2404.51,-599.3 2410,-595.5 2430.74,-581.14 2447.48,-557.63 2458.31,-539.46\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2461.5,-540.94 2463.43,-530.52 2455.42,-537.46 2461.5,-540.94\"/>\n",
       "</g>\n",
       "<!-- 140404045524624tanh&#45;&gt;140404045524624 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>140404045524624tanh&#45;&gt;140404045524624</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2178.52,-667.29C2190.32,-661.91 2205.3,-655.42 2219,-650.5 2224.61,-648.49 2230.44,-646.52 2236.34,-644.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2237.8,-647.84 2246.29,-641.51 2235.7,-641.16 2237.8,-647.84\"/>\n",
       "</g>\n",
       "<!-- 140404120178320 -->\n",
       "<g id=\"node94\" class=\"node\">\n",
       "<title>140404120178320</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-495.5 638.5,-531.5 820.5,-531.5 820.5,-495.5 638.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"648.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-495.5 658.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.9790</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-495.5 738.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0029</text>\n",
       "</g>\n",
       "<!-- 140404120148880* -->\n",
       "<g id=\"node178\" class=\"node\">\n",
       "<title>140404120148880*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"888\" cy=\"-513.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120178320&#45;&gt;140404120148880* -->\n",
       "<g id=\"edge107\" class=\"edge\">\n",
       "<title>140404120178320&#45;&gt;140404120148880*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M820.63,-513.5C831.18,-513.5 841.43,-513.5 850.6,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"850.82,-517 860.82,-513.5 850.82,-510 850.82,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120243920 -->\n",
       "<g id=\"node95\" class=\"node\">\n",
       "<title>140404120243920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2855.5,-275.5 2855.5,-311.5 3041.5,-311.5 3041.5,-275.5 2855.5,-275.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2865.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2875.5,-275.5 2875.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0126</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2955.5,-275.5 2955.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2998.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120243920&#45;&gt;140404120244176+ -->\n",
       "<g id=\"edge92\" class=\"edge\">\n",
       "<title>140404120243920&#45;&gt;140404120244176+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3028.19,-311.54C3033.75,-314.14 3039.08,-317.11 3044,-320.5 3065.15,-335.09 3082.01,-359.24 3092.77,-377.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3089.76,-379.54 3097.7,-386.56 3095.87,-376.13 3089.76,-379.54\"/>\n",
       "</g>\n",
       "<!-- 140404120243920*&#45;&gt;140404120243920 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>140404120243920*&#45;&gt;140404120243920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-293.5C2825.35,-293.5 2834.89,-293.5 2845.03,-293.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2845.35,-297 2855.35,-293.5 2845.35,-290 2845.35,-297\"/>\n",
       "</g>\n",
       "<!-- 140404120260368 -->\n",
       "<g id=\"node97\" class=\"node\">\n",
       "<title>140404120260368</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-770.5 2536,-806.5 2727,-806.5 2727,-770.5 2536,-770.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-770.5 2556,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7646</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-770.5 2641,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120260368&#45;&gt;140404120268880+ -->\n",
       "<g id=\"edge160\" class=\"edge\">\n",
       "<title>140404120260368&#45;&gt;140404120268880+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2727.15,-788.5C2736.09,-788.5 2744.73,-788.5 2752.57,-788.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.73,-792 2762.73,-788.5 2752.73,-785 2752.73,-792\"/>\n",
       "</g>\n",
       "<!-- 140404120260368*&#45;&gt;140404120260368 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>140404120260368*&#45;&gt;140404120260368</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-788.5C2507.67,-788.5 2516.31,-788.5 2525.51,-788.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2525.68,-792 2535.68,-788.5 2525.68,-785 2525.68,-792\"/>\n",
       "</g>\n",
       "<!-- 140404120154000 -->\n",
       "<g id=\"node99\" class=\"node\">\n",
       "<title>140404120154000</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-385.5 953,-421.5 1140,-421.5 1140,-385.5 953,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-385.5 973,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-385.5 1058,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0008</text>\n",
       "</g>\n",
       "<!-- 140404120154000&#45;&gt;140404120152656* -->\n",
       "<g id=\"edge115\" class=\"edge\">\n",
       "<title>140404120154000&#45;&gt;140404120152656*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1115.25,-421.56C1124.31,-424.35 1133.43,-427.36 1142,-430.5 1152.5,-434.35 1163.72,-439.18 1173.69,-443.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1172.26,-446.94 1182.8,-448 1175.22,-440.6 1172.26,-446.94\"/>\n",
       "</g>\n",
       "<!-- 140404120178640 -->\n",
       "<g id=\"node100\" class=\"node\">\n",
       "<title>140404120178640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"636,-550.5 636,-586.5 823,-586.5 823,-550.5 636,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"646\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"656,-550.5 656,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1102</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"741,-550.5 741,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"782\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120178640&#45;&gt;140404120148176+ -->\n",
       "<g id=\"edge111\" class=\"edge\">\n",
       "<title>140404120178640&#45;&gt;140404120148176+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M823.35,-568.5C832.97,-568.5 842.29,-568.5 850.7,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"850.81,-572 860.81,-568.5 850.81,-565 850.81,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120244176 -->\n",
       "<g id=\"node101\" class=\"node\">\n",
       "<title>140404120244176</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3170,-385.5 3170,-421.5 3361,-421.5 3361,-385.5 3170,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3180\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3190,-385.5 3190,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3232.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6127</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3275,-385.5 3275,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3318\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120244176&#45;&gt;140404120256656tanh -->\n",
       "<g id=\"edge110\" class=\"edge\">\n",
       "<title>140404120244176&#45;&gt;140404120256656tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3361.15,-403.5C3370.09,-403.5 3378.73,-403.5 3386.57,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3386.73,-407 3396.73,-403.5 3386.73,-400 3386.73,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120244176+&#45;&gt;140404120244176 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>140404120244176+&#45;&gt;140404120244176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3134.19,-403.5C3141.67,-403.5 3150.31,-403.5 3159.51,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3159.68,-407 3169.68,-403.5 3159.68,-400 3159.68,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120268880 -->\n",
       "<g id=\"node103\" class=\"node\">\n",
       "<title>140404120268880</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2853,-742.5 2853,-778.5 3044,-778.5 3044,-742.5 2853,-742.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2863\" y=\"-756.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2873,-742.5 2873,-778.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-756.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.2584</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2958,-742.5 2958,-778.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-756.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120269392+ -->\n",
       "<g id=\"node117\" class=\"node\">\n",
       "<title>140404120269392+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3107\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3107\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120268880&#45;&gt;140404120269392+ -->\n",
       "<g id=\"edge114\" class=\"edge\">\n",
       "<title>140404120268880&#45;&gt;140404120269392+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2988.13,-742.49C3006.21,-732.99 3027.38,-720.23 3044,-705.5 3062.65,-688.97 3079.41,-666.07 3090.78,-648.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3093.8,-650.45 3096.22,-640.14 3087.89,-646.69 3093.8,-650.45\"/>\n",
       "</g>\n",
       "<!-- 140404120268880+&#45;&gt;140404120268880 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>140404120268880+&#45;&gt;140404120268880</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2816.14,-784C2823.99,-782.6 2833.17,-780.95 2842.99,-779.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2843.75,-782.62 2852.98,-777.41 2842.52,-775.73 2843.75,-782.62\"/>\n",
       "</g>\n",
       "<!-- 140404120285328 -->\n",
       "<g id=\"node105\" class=\"node\">\n",
       "<title>140404120285328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4681,-412.5 4681,-448.5 4863,-448.5 4863,-412.5 4681,-412.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4691\" y=\"-426.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4701,-412.5 4701,-448.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4741\" y=\"-426.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.8690</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4781,-412.5 4781,-448.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4822\" y=\"-426.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120285840+ -->\n",
       "<g id=\"node119\" class=\"node\">\n",
       "<title>140404120285840+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"4926\" cy=\"-430.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"4926\" y=\"-426.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120285328&#45;&gt;140404120285840+ -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>140404120285328&#45;&gt;140404120285840+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4863.18,-430.5C4872.14,-430.5 4880.83,-430.5 4888.73,-430.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4888.96,-434 4898.96,-430.5 4888.96,-427 4888.96,-434\"/>\n",
       "</g>\n",
       "<!-- 140404120285328+ -->\n",
       "<g id=\"node106\" class=\"node\">\n",
       "<title>140404120285328+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"4554\" cy=\"-430.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"4554\" y=\"-426.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120285328+&#45;&gt;140404120285328 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>140404120285328+&#45;&gt;140404120285328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4581.05,-430.5C4603.53,-430.5 4637.48,-430.5 4670.5,-430.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4670.7,-434 4680.7,-430.5 4670.7,-427 4670.7,-434\"/>\n",
       "</g>\n",
       "<!-- 140404120211728 -->\n",
       "<g id=\"node107\" class=\"node\">\n",
       "<title>140404120211728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-701.5 638.5,-737.5 820.5,-737.5 820.5,-701.5 638.5,-701.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"648.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-701.5 658.5,-737.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.2212</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-701.5 738.5,-737.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0025</text>\n",
       "</g>\n",
       "<!-- 140404120211728&#45;&gt;140404120151056* -->\n",
       "<g id=\"edge116\" class=\"edge\">\n",
       "<title>140404120211728&#45;&gt;140404120151056*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M798.25,-737.56C807.31,-740.35 816.43,-743.36 825,-746.5 835.5,-750.35 846.72,-755.18 856.69,-759.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"855.26,-762.94 865.8,-764 858.22,-756.6 855.26,-762.94\"/>\n",
       "</g>\n",
       "<!-- 140404120211792 -->\n",
       "<g id=\"node108\" class=\"node\">\n",
       "<title>140404120211792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-605.5 953,-641.5 1140,-641.5 1140,-605.5 953,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-605.5 973,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8692</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-605.5 1058,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0037</text>\n",
       "</g>\n",
       "<!-- 140404120211792&#45;&gt;140404120085136* -->\n",
       "<g id=\"edge128\" class=\"edge\">\n",
       "<title>140404120211792&#45;&gt;140404120085136*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1115.25,-641.56C1124.31,-644.35 1133.43,-647.36 1142,-650.5 1152.5,-654.35 1163.72,-659.18 1173.69,-663.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1172.26,-666.94 1182.8,-668 1175.22,-660.6 1172.26,-666.94\"/>\n",
       "</g>\n",
       "<!-- 140404120269136 -->\n",
       "<g id=\"node109\" class=\"node\">\n",
       "<title>140404120269136</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2853,-550.5 2853,-586.5 3044,-586.5 3044,-550.5 2853,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2863\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2873,-550.5 2873,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2915.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.2216</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2958,-550.5 2958,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120269136&#45;&gt;140404120269392+ -->\n",
       "<g id=\"edge103\" class=\"edge\">\n",
       "<title>140404120269136&#45;&gt;140404120269392+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3017.25,-586.56C3026.31,-589.35 3035.43,-592.36 3044,-595.5 3054.5,-599.35 3065.72,-604.18 3075.69,-608.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3074.26,-611.94 3084.8,-613 3077.22,-605.6 3074.26,-611.94\"/>\n",
       "</g>\n",
       "<!-- 140404120269136*&#45;&gt;140404120269136 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>140404120269136*&#45;&gt;140404120269136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-568.5C2824.67,-568.5 2833.31,-568.5 2842.51,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2842.68,-572 2852.68,-568.5 2842.68,-565 2842.68,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120211856 -->\n",
       "<g id=\"node111\" class=\"node\">\n",
       "<title>140404120211856</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1268,-605.5 1268,-641.5 1459,-641.5 1459,-605.5 1268,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1278\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1288,-605.5 1288,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1330.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5151</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1373,-605.5 1373,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0012</text>\n",
       "</g>\n",
       "<!-- 140404120211856&#45;&gt;140404120088016* -->\n",
       "<g id=\"edge189\" class=\"edge\">\n",
       "<title>140404120211856&#45;&gt;140404120088016*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1459.15,-623.5C1468.09,-623.5 1476.73,-623.5 1484.57,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1484.73,-627 1494.73,-623.5 1484.73,-620 1484.73,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120285584 -->\n",
       "<g id=\"node112\" class=\"node\">\n",
       "<title>140404120285584</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4463,-490.5 4463,-526.5 4645,-526.5 4645,-490.5 4463,-490.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4473\" y=\"-504.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4483,-490.5 4483,-526.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4523\" y=\"-504.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7077</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4563,-490.5 4563,-526.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4604\" y=\"-504.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120285584&#45;&gt;140404120285840+ -->\n",
       "<g id=\"edge179\" class=\"edge\">\n",
       "<title>140404120285584&#45;&gt;140404120285840+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4645.27,-499.5C4707.37,-491.87 4791.27,-478.7 4863,-457.5 4873.66,-454.35 4884.93,-449.83 4894.88,-445.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4896.35,-448.57 4903.97,-441.21 4893.42,-442.21 4896.35,-448.57\"/>\n",
       "</g>\n",
       "<!-- 140404120285584* -->\n",
       "<g id=\"node113\" class=\"node\">\n",
       "<title>140404120285584*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3741\" cy=\"-568.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3741\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120285584*&#45;&gt;140404120285584 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>140404120285584*&#45;&gt;140404120285584</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3768.06,-566.57C3872.69,-558.83 4265.51,-529.77 4452.65,-515.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4453.06,-519.4 4462.77,-515.18 4452.54,-512.42 4453.06,-519.4\"/>\n",
       "</g>\n",
       "<!-- 140404120211920 -->\n",
       "<g id=\"node114\" class=\"node\">\n",
       "<title>140404120211920</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-715.5 953,-751.5 1140,-751.5 1140,-715.5 953,-715.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-715.5 973,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1588</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-715.5 1058,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0012</text>\n",
       "</g>\n",
       "<!-- 140404120211920&#45;&gt;140404045178448+ -->\n",
       "<g id=\"edge85\" class=\"edge\">\n",
       "<title>140404120211920&#45;&gt;140404045178448+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1140.35,-733.5C1149.97,-733.5 1159.29,-733.5 1167.7,-733.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.81,-737 1177.81,-733.5 1167.81,-730 1167.81,-737\"/>\n",
       "</g>\n",
       "<!-- 140404120212048 -->\n",
       "<g id=\"node115\" class=\"node\">\n",
       "<title>140404120212048</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-83.5 953,-119.5 1140,-119.5 1140,-83.5 953,-83.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-83.5 973,-119.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9217</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-83.5 1058,-119.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0136</text>\n",
       "</g>\n",
       "<!-- 140404120001744* -->\n",
       "<g id=\"node187\" class=\"node\">\n",
       "<title>140404120001744*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1205\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120212048&#45;&gt;140404120001744* -->\n",
       "<g id=\"edge183\" class=\"edge\">\n",
       "<title>140404120212048&#45;&gt;140404120001744*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1140.35,-84.89C1150.36,-83.1 1160.04,-81.36 1168.71,-79.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1169.5,-83.23 1178.73,-78.02 1168.27,-76.34 1169.5,-83.23\"/>\n",
       "</g>\n",
       "<!-- 140404120269392 -->\n",
       "<g id=\"node116\" class=\"node\">\n",
       "<title>140404120269392</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3170,-605.5 3170,-641.5 3361,-641.5 3361,-605.5 3170,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3180\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3190,-605.5 3190,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3232.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.4800</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3275,-605.5 3275,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3318\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120269520tanh -->\n",
       "<g id=\"node123\" class=\"node\">\n",
       "<title>140404120269520tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3424\" cy=\"-623.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3424\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120269392&#45;&gt;140404120269520tanh -->\n",
       "<g id=\"edge204\" class=\"edge\">\n",
       "<title>140404120269392&#45;&gt;140404120269520tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3361.15,-623.5C3370.09,-623.5 3378.73,-623.5 3386.57,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3386.73,-627 3396.73,-623.5 3386.73,-620 3386.73,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120269392+&#45;&gt;140404120269392 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>140404120269392+&#45;&gt;140404120269392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3134.19,-623.5C3141.67,-623.5 3150.31,-623.5 3159.51,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3159.68,-627 3169.68,-623.5 3159.68,-620 3159.68,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120285840 -->\n",
       "<g id=\"node118\" class=\"node\">\n",
       "<title>140404120285840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4989,-381.5 4989,-417.5 5171,-417.5 5171,-381.5 4989,-381.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4999\" y=\"-395.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5009,-381.5 5009,-417.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5049\" y=\"-395.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.5767</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5089,-381.5 5089,-417.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5130\" y=\"-395.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120286352+ -->\n",
       "<g id=\"node133\" class=\"node\">\n",
       "<title>140404120286352+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"5234\" cy=\"-375.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"5234\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120285840&#45;&gt;140404120286352+ -->\n",
       "<g id=\"edge138\" class=\"edge\">\n",
       "<title>140404120285840&#45;&gt;140404120286352+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5171.18,-385.26C5180.42,-383.8 5189.38,-382.39 5197.47,-381.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5198.23,-384.53 5207.56,-379.52 5197.14,-377.62 5198.23,-384.53\"/>\n",
       "</g>\n",
       "<!-- 140404120285840+&#45;&gt;140404120285840 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>140404120285840+&#45;&gt;140404120285840</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4952.1,-425.38C4960.34,-423.7 4970.06,-421.72 4980.4,-419.61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4981.37,-422.98 4990.47,-417.56 4979.97,-416.12 4981.37,-422.98\"/>\n",
       "</g>\n",
       "<!-- 140404120212112 -->\n",
       "<g id=\"node120\" class=\"node\">\n",
       "<title>140404120212112</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1272.5,-165.5 1272.5,-201.5 1454.5,-201.5 1454.5,-165.5 1272.5,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1292.5,-165.5 1292.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6165</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1372.5,-165.5 1372.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0205</text>\n",
       "</g>\n",
       "<!-- 140404120212112&#45;&gt;140404120240784* -->\n",
       "<g id=\"edge147\" class=\"edge\">\n",
       "<title>140404120212112&#45;&gt;140404120240784*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1429.15,-165.49C1439.21,-162.36 1449.43,-158.99 1459,-155.5 1469.26,-151.76 1480.25,-147.18 1490.08,-142.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1491.76,-145.94 1499.46,-138.66 1488.9,-139.55 1491.76,-145.94\"/>\n",
       "</g>\n",
       "<!-- 140404120212176 -->\n",
       "<g id=\"node121\" class=\"node\">\n",
       "<title>140404120212176</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1587.5,-165.5 1587.5,-201.5 1773.5,-201.5 1773.5,-165.5 1587.5,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1597.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1607.5,-165.5 1607.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1647.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.3339</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1687.5,-165.5 1687.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120212176&#45;&gt;140404120241616* -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>140404120212176&#45;&gt;140404120241616*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1773.89,-183.5C1783.67,-183.5 1793.13,-183.5 1801.66,-183.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.91,-187 1811.91,-183.5 1801.91,-180 1801.91,-187\"/>\n",
       "</g>\n",
       "<!-- 140404120269520 -->\n",
       "<g id=\"node122\" class=\"node\">\n",
       "<title>140404120269520</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-605.5 3487,-641.5 3678,-641.5 3678,-605.5 3487,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-605.5 3507,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9015</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-605.5 3592,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0595</text>\n",
       "</g>\n",
       "<!-- 140404120269520&#45;&gt;140404120285584* -->\n",
       "<g id=\"edge99\" class=\"edge\">\n",
       "<title>140404120269520&#45;&gt;140404120285584*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3648.15,-605.49C3658.21,-602.36 3668.43,-598.99 3678,-595.5 3688.26,-591.76 3699.25,-587.18 3709.08,-582.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3710.76,-585.94 3718.46,-578.66 3707.9,-579.55 3710.76,-585.94\"/>\n",
       "</g>\n",
       "<!-- 140404120269520tanh&#45;&gt;140404120269520 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>140404120269520tanh&#45;&gt;140404120269520</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3451.19,-623.5C3458.67,-623.5 3467.31,-623.5 3476.51,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3476.68,-627 3486.68,-623.5 3476.68,-620 3476.68,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120212240 -->\n",
       "<g id=\"node124\" class=\"node\">\n",
       "<title>140404120212240</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1272.5,-0.5 1272.5,-36.5 1454.5,-36.5 1454.5,-0.5 1272.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1292.5,-0.5 1292.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6716</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1372.5,-0.5 1372.5,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404045641552+ -->\n",
       "<g id=\"node150\" class=\"node\">\n",
       "<title>140404045641552+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1522\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1522\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120212240&#45;&gt;140404045641552+ -->\n",
       "<g id=\"edge185\" class=\"edge\">\n",
       "<title>140404120212240&#45;&gt;140404045641552+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1432.25,-36.56C1441.31,-39.35 1450.43,-42.36 1459,-45.5 1469.5,-49.35 1480.72,-54.18 1490.69,-58.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1489.26,-61.94 1499.8,-63 1492.22,-55.6 1489.26,-61.94\"/>\n",
       "</g>\n",
       "<!-- 140404120286096 -->\n",
       "<g id=\"node125\" class=\"node\">\n",
       "<title>140404120286096</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4022,-307.5 4022,-343.5 4209,-343.5 4209,-307.5 4022,-307.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4032\" y=\"-321.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4042,-307.5 4042,-343.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4084.5\" y=\"-321.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6131</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4127,-307.5 4127,-343.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4168\" y=\"-321.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120286096&#45;&gt;140404120286352+ -->\n",
       "<g id=\"edge180\" class=\"edge\">\n",
       "<title>140404120286096&#45;&gt;140404120286352+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4209.21,-334.57C4297.5,-342.41 4434.18,-352.5 4553,-352.5 4553,-352.5 4553,-352.5 4927,-352.5 5025.05,-352.5 5140.28,-364.39 5197.3,-371.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5196.91,-374.55 5207.26,-372.25 5197.74,-367.6 5196.91,-374.55\"/>\n",
       "</g>\n",
       "<!-- 140404120286096* -->\n",
       "<g id=\"node126\" class=\"node\">\n",
       "<title>140404120286096*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3741\" cy=\"-293.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3741\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140404120286096*&#45;&gt;140404120286096 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>140404120286096*&#45;&gt;140404120286096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3767.9,-295.72C3817.6,-299.99 3928.81,-309.55 4011.94,-316.69\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4011.67,-320.18 4021.93,-317.55 4012.26,-313.2 4011.67,-320.18\"/>\n",
       "</g>\n",
       "<!-- 140404120212432 -->\n",
       "<g id=\"node127\" class=\"node\">\n",
       "<title>140404120212432</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1589.5,-550.5 1589.5,-586.5 1771.5,-586.5 1771.5,-550.5 1589.5,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1599.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1609.5,-550.5 1609.5,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7479</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1689.5,-550.5 1689.5,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0220</text>\n",
       "</g>\n",
       "<!-- 140404120212432&#45;&gt;140404120242384* -->\n",
       "<g id=\"edge199\" class=\"edge\">\n",
       "<title>140404120212432&#45;&gt;140404120242384*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1771.63,-568.5C1782.18,-568.5 1792.43,-568.5 1801.6,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.82,-572 1811.82,-568.5 1801.82,-565 1801.82,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120212496 -->\n",
       "<g id=\"node128\" class=\"node\">\n",
       "<title>140404120212496</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904.5,-385.5 1904.5,-421.5 2090.5,-421.5 2090.5,-385.5 1904.5,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924.5,-385.5 1924.5,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6914</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2004.5,-385.5 2004.5,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0179</text>\n",
       "</g>\n",
       "<!-- 140404120212496&#45;&gt;140404120242896* -->\n",
       "<g id=\"edge146\" class=\"edge\">\n",
       "<title>140404120212496&#45;&gt;140404120242896*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2090.89,-403.5C2100.67,-403.5 2110.13,-403.5 2118.66,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.91,-407 2128.91,-403.5 2118.91,-400 2118.91,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120212560 -->\n",
       "<g id=\"node129\" class=\"node\">\n",
       "<title>140404120212560</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221,-550.5 2221,-586.5 2408,-586.5 2408,-550.5 2221,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241,-550.5 2241,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.3720</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2326,-550.5 2326,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0221</text>\n",
       "</g>\n",
       "<!-- 140404120212560&#45;&gt;140404120243408* -->\n",
       "<g id=\"edge117\" class=\"edge\">\n",
       "<title>140404120212560&#45;&gt;140404120243408*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2383.25,-586.56C2392.31,-589.35 2401.43,-592.36 2410,-595.5 2420.5,-599.35 2431.72,-604.18 2441.69,-608.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2440.26,-611.94 2450.8,-613 2443.22,-605.6 2440.26,-611.94\"/>\n",
       "</g>\n",
       "<!-- 140404120269904 -->\n",
       "<g id=\"node130\" class=\"node\">\n",
       "<title>140404120269904</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1906.5,-275.5 1906.5,-311.5 2088.5,-311.5 2088.5,-275.5 1906.5,-275.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1916.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1926.5,-275.5 1926.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8963</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2006.5,-275.5 2006.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120270160+ -->\n",
       "<g id=\"node140\" class=\"node\">\n",
       "<title>140404120270160+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2156\" cy=\"-293.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2156\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120269904&#45;&gt;140404120270160+ -->\n",
       "<g id=\"edge175\" class=\"edge\">\n",
       "<title>140404120269904&#45;&gt;140404120270160+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2088.63,-293.5C2099.18,-293.5 2109.43,-293.5 2118.6,-293.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.82,-297 2128.82,-293.5 2118.82,-290 2118.82,-297\"/>\n",
       "</g>\n",
       "<!-- 140404120269904*&#45;&gt;140404120269904 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>140404120269904*&#45;&gt;140404120269904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1853.3,-361.08C1864.96,-348.47 1882.97,-331.05 1902,-320.5 1905.08,-318.79 1908.27,-317.19 1911.55,-315.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1913.13,-318.8 1920.95,-311.66 1910.38,-312.37 1913.13,-318.8\"/>\n",
       "</g>\n",
       "<!-- 140404120286352 -->\n",
       "<g id=\"node132\" class=\"node\">\n",
       "<title>140404120286352</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"5297,-357.5 5297,-393.5 5479,-393.5 5479,-357.5 5297,-357.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"5307\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5317,-357.5 5317,-393.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5357\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.9636</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5397,-357.5 5397,-393.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5438\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120286480tanh -->\n",
       "<g id=\"node137\" class=\"node\">\n",
       "<title>140404120286480tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"5542\" cy=\"-375.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"5542\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120286352&#45;&gt;140404120286480tanh -->\n",
       "<g id=\"edge131\" class=\"edge\">\n",
       "<title>140404120286352&#45;&gt;140404120286480tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5479.18,-375.5C5488.14,-375.5 5496.83,-375.5 5504.73,-375.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5504.96,-379 5514.96,-375.5 5504.96,-372 5504.96,-379\"/>\n",
       "</g>\n",
       "<!-- 140404120286352+&#45;&gt;140404120286352 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>140404120286352+&#45;&gt;140404120286352</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5261.13,-375.5C5268.67,-375.5 5277.39,-375.5 5286.64,-375.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5286.87,-379 5296.87,-375.5 5286.87,-372 5286.87,-379\"/>\n",
       "</g>\n",
       "<!-- 140404120212624 -->\n",
       "<g id=\"node134\" class=\"node\">\n",
       "<title>140404120212624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2538.5,-220.5 2538.5,-256.5 2724.5,-256.5 2724.5,-220.5 2538.5,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2548.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2558.5,-220.5 2558.5,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0381</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2638.5,-220.5 2638.5,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2681.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0077</text>\n",
       "</g>\n",
       "<!-- 140404120212624&#45;&gt;140404120243920* -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>140404120212624&#45;&gt;140404120243920*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2700.25,-256.56C2709.31,-259.35 2718.43,-262.36 2727,-265.5 2737.5,-269.35 2748.72,-274.18 2758.69,-278.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2757.26,-281.94 2767.8,-283 2760.22,-275.6 2757.26,-281.94\"/>\n",
       "</g>\n",
       "<!-- 140404120212688 -->\n",
       "<g id=\"node135\" class=\"node\">\n",
       "<title>140404120212688</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-440.5 1902,-476.5 2093,-476.5 2093,-440.5 1902,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-440.5 1922,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8036</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-440.5 2007,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0233</text>\n",
       "</g>\n",
       "<!-- 140404120212688&#45;&gt;140404120242640+ -->\n",
       "<g id=\"edge186\" class=\"edge\">\n",
       "<title>140404120212688&#45;&gt;140404120242640+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2093.15,-458.5C2102.09,-458.5 2110.73,-458.5 2118.57,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.73,-462 2128.73,-458.5 2118.73,-455 2118.73,-462\"/>\n",
       "</g>\n",
       "<!-- 140404120286480 -->\n",
       "<g id=\"node136\" class=\"node\">\n",
       "<title>140404120286480</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"5605,-357.5 5605,-393.5 5787,-393.5 5787,-357.5 5605,-357.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"5615\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5625,-357.5 5625,-393.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5665\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.9614</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"5705,-357.5 5705,-393.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"5746\" y=\"-371.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140404120286480tanh&#45;&gt;140404120286480 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>140404120286480tanh&#45;&gt;140404120286480</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5569.13,-375.5C5576.67,-375.5 5585.39,-375.5 5594.64,-375.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5594.87,-379 5604.87,-375.5 5594.87,-372 5594.87,-379\"/>\n",
       "</g>\n",
       "<!-- 140404120212816 -->\n",
       "<g id=\"node138\" class=\"node\">\n",
       "<title>140404120212816</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1589.5,-838.5 1589.5,-874.5 1771.5,-874.5 1771.5,-838.5 1589.5,-838.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1599.5\" y=\"-852.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1609.5,-838.5 1609.5,-874.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-852.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.1433</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1689.5,-838.5 1689.5,-874.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1730.5\" y=\"-852.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0175</text>\n",
       "</g>\n",
       "<!-- 140404120212816&#45;&gt;140404120257040* -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>140404120212816&#45;&gt;140404120257040*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1771.63,-849.01C1782.39,-848.12 1792.85,-847.25 1802.16,-846.48\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1802.46,-849.96 1812.13,-845.65 1801.88,-842.99 1802.46,-849.96\"/>\n",
       "</g>\n",
       "<!-- 140404120270160 -->\n",
       "<g id=\"node139\" class=\"node\">\n",
       "<title>140404120270160</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2223.5,-275.5 2223.5,-311.5 2405.5,-311.5 2405.5,-275.5 2223.5,-275.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2233.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2243.5,-275.5 2243.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0419</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2323.5,-275.5 2323.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120270672+ -->\n",
       "<g id=\"node153\" class=\"node\">\n",
       "<title>140404120270672+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2473\" cy=\"-348.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2473\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120270160&#45;&gt;140404120270672+ -->\n",
       "<g id=\"edge163\" class=\"edge\">\n",
       "<title>140404120270160&#45;&gt;140404120270672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2383.25,-311.56C2392.31,-314.35 2401.43,-317.36 2410,-320.5 2420.5,-324.35 2431.72,-329.18 2441.69,-333.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2440.26,-336.94 2450.8,-338 2443.22,-330.6 2440.26,-336.94\"/>\n",
       "</g>\n",
       "<!-- 140404120270160+&#45;&gt;140404120270160 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>140404120270160+&#45;&gt;140404120270160</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-293.5C2191.92,-293.5 2202.23,-293.5 2213.17,-293.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2213.36,-297 2223.36,-293.5 2213.36,-290 2213.36,-297\"/>\n",
       "</g>\n",
       "<!-- 140404120212880 -->\n",
       "<g id=\"node141\" class=\"node\">\n",
       "<title>140404120212880</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-605.5 1902,-641.5 2093,-641.5 2093,-605.5 1902,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-605.5 1922,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9905</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-605.5 2007,-641.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-619.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0143</text>\n",
       "</g>\n",
       "<!-- 140404120212880&#45;&gt;140404120257552* -->\n",
       "<g id=\"edge120\" class=\"edge\">\n",
       "<title>140404120212880&#45;&gt;140404120257552*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2093.15,-623.5C2102.09,-623.5 2110.73,-623.5 2118.57,-623.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.73,-627 2128.73,-623.5 2118.73,-620 2118.73,-627\"/>\n",
       "</g>\n",
       "<!-- 140404120212944 -->\n",
       "<g id=\"node142\" class=\"node\">\n",
       "<title>140404120212944</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2223.5,-660.5 2223.5,-696.5 2405.5,-696.5 2405.5,-660.5 2223.5,-660.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2233.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2243.5,-660.5 2243.5,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.2327</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2323.5,-660.5 2323.5,-696.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-674.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0175</text>\n",
       "</g>\n",
       "<!-- 140404120212944&#45;&gt;140404120258064* -->\n",
       "<g id=\"edge171\" class=\"edge\">\n",
       "<title>140404120212944&#45;&gt;140404120258064*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2405.63,-678.5C2416.18,-678.5 2426.43,-678.5 2435.6,-678.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.82,-682 2445.82,-678.5 2435.82,-675 2435.82,-682\"/>\n",
       "</g>\n",
       "<!-- 140404120213008 -->\n",
       "<g id=\"node143\" class=\"node\">\n",
       "<title>140404120213008</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-495.5 2536,-531.5 2727,-531.5 2727,-495.5 2536,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-495.5 2556,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.5441</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-495.5 2641,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0061</text>\n",
       "</g>\n",
       "<!-- 140404120213008&#45;&gt;140404120258576* -->\n",
       "<g id=\"edge86\" class=\"edge\">\n",
       "<title>140404120213008&#45;&gt;140404120258576*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2727.15,-513.5C2736.09,-513.5 2744.73,-513.5 2752.57,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.73,-517 2762.73,-513.5 2752.73,-510 2752.73,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120213072 -->\n",
       "<g id=\"node144\" class=\"node\">\n",
       "<title>140404120213072</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904.5,-770.5 1904.5,-806.5 2090.5,-806.5 2090.5,-770.5 1904.5,-770.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924.5,-770.5 1924.5,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0904</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2004.5,-770.5 2004.5,-806.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2047.5\" y=\"-784.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0185</text>\n",
       "</g>\n",
       "<!-- 140404120213072&#45;&gt;140404120257296+ -->\n",
       "<g id=\"edge136\" class=\"edge\">\n",
       "<title>140404120213072&#45;&gt;140404120257296+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2066.25,-806.56C2075.31,-809.35 2084.43,-812.36 2093,-815.5 2103.5,-819.35 2114.72,-824.18 2124.69,-828.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2123.26,-831.94 2133.8,-833 2126.22,-825.6 2123.26,-831.94\"/>\n",
       "</g>\n",
       "<!-- 140404120270416 -->\n",
       "<g id=\"node145\" class=\"node\">\n",
       "<title>140404120270416</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221,-330.5 2221,-366.5 2408,-366.5 2408,-330.5 2221,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241,-330.5 2241,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.2788</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2326,-330.5 2326,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120270416&#45;&gt;140404120270672+ -->\n",
       "<g id=\"edge121\" class=\"edge\">\n",
       "<title>140404120270416&#45;&gt;140404120270672+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2408.35,-348.5C2417.97,-348.5 2427.29,-348.5 2435.7,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.81,-352 2445.81,-348.5 2435.81,-345 2435.81,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120270416*&#45;&gt;140404120270416 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>140404120270416*&#45;&gt;140404120270416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2183.19,-348.5C2191.22,-348.5 2200.59,-348.5 2210.55,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2210.68,-352 2220.68,-348.5 2210.68,-345 2210.68,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120213200 -->\n",
       "<g id=\"node147\" class=\"node\">\n",
       "<title>140404120213200</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1587,-920.5 1587,-956.5 1774,-956.5 1774,-920.5 1587,-920.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1597\" y=\"-934.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1607,-920.5 1607,-956.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-934.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6263</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1692,-920.5 1692,-956.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-934.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0105</text>\n",
       "</g>\n",
       "<!-- 140404120213200&#45;&gt;140404120259344* -->\n",
       "<g id=\"edge172\" class=\"edge\">\n",
       "<title>140404120213200&#45;&gt;140404120259344*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1774.35,-938.5C1783.97,-938.5 1793.29,-938.5 1801.7,-938.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.81,-942 1811.81,-938.5 1801.81,-935 1801.81,-942\"/>\n",
       "</g>\n",
       "<!-- 140404120213264 -->\n",
       "<g id=\"node148\" class=\"node\">\n",
       "<title>140404120213264</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-715.5 1902,-751.5 2093,-751.5 2093,-715.5 1902,-715.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-715.5 1922,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7687</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-715.5 2007,-751.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0086</text>\n",
       "</g>\n",
       "<!-- 140404120213264&#45;&gt;140404120259856* -->\n",
       "<g id=\"edge124\" class=\"edge\">\n",
       "<title>140404120213264&#45;&gt;140404120259856*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2093.15,-733.5C2102.09,-733.5 2110.73,-733.5 2118.57,-733.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.73,-737 2128.73,-733.5 2118.73,-730 2118.73,-737\"/>\n",
       "</g>\n",
       "<!-- 140404045641552 -->\n",
       "<g id=\"node149\" class=\"node\">\n",
       "<title>140404045641552</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1587,-55.5 1587,-91.5 1774,-91.5 1774,-55.5 1587,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1597\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1607,-55.5 1607,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1649.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.1717</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1692,-55.5 1692,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404045641552&#45;&gt;140404120241296+ -->\n",
       "<g id=\"edge164\" class=\"edge\">\n",
       "<title>140404045641552&#45;&gt;140404120241296+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1749.25,-91.56C1758.31,-94.35 1767.43,-97.36 1776,-100.5 1786.5,-104.35 1797.72,-109.18 1807.69,-113.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1806.26,-116.94 1816.8,-118 1809.22,-110.6 1806.26,-116.94\"/>\n",
       "</g>\n",
       "<!-- 140404045641552+&#45;&gt;140404045641552 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>140404045641552+&#45;&gt;140404045641552</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1549.19,-73.5C1557.22,-73.5 1566.59,-73.5 1576.55,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1576.68,-77 1586.68,-73.5 1576.68,-70 1576.68,-77\"/>\n",
       "</g>\n",
       "<!-- 140404120213328 -->\n",
       "<g id=\"node151\" class=\"node\">\n",
       "<title>140404120213328</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2223.5,-880.5 2223.5,-916.5 2405.5,-916.5 2405.5,-880.5 2223.5,-880.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2233.5\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2243.5,-880.5 2243.5,-916.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2283.5\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8067</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2323.5,-880.5 2323.5,-916.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0106</text>\n",
       "</g>\n",
       "<!-- 140404120213328&#45;&gt;140404120260368* -->\n",
       "<g id=\"edge151\" class=\"edge\">\n",
       "<title>140404120213328&#45;&gt;140404120260368*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2395.37,-880.33C2400.53,-877.77 2405.47,-874.84 2410,-871.5 2433.52,-854.18 2428,-839.51 2446,-816.5 2447.4,-814.72 2448.88,-812.91 2450.41,-811.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2453.17,-813.28 2457.21,-803.49 2447.94,-808.63 2453.17,-813.28\"/>\n",
       "</g>\n",
       "<!-- 140404120270672 -->\n",
       "<g id=\"node152\" class=\"node\">\n",
       "<title>140404120270672</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2538,-330.5 2538,-366.5 2725,-366.5 2725,-330.5 2538,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2548\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2558,-330.5 2558,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2600.5\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.2369</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2643,-330.5 2643,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120271184+ -->\n",
       "<g id=\"node165\" class=\"node\">\n",
       "<title>140404120271184+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2790\" cy=\"-348.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2790\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120270672&#45;&gt;140404120271184+ -->\n",
       "<g id=\"edge109\" class=\"edge\">\n",
       "<title>140404120270672&#45;&gt;140404120271184+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2725.35,-348.5C2734.97,-348.5 2744.29,-348.5 2752.7,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.81,-352 2762.81,-348.5 2752.81,-345 2752.81,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120270672+&#45;&gt;140404120270672 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>140404120270672+&#45;&gt;140404120270672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2500.19,-348.5C2508.22,-348.5 2517.59,-348.5 2527.55,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2527.68,-352 2537.68,-348.5 2527.68,-345 2527.68,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120213392 -->\n",
       "<g id=\"node154\" class=\"node\">\n",
       "<title>140404120213392</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2536,-550.5 2536,-586.5 2727,-586.5 2727,-550.5 2536,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2546\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2556,-550.5 2556,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2598.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6695</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2641,-550.5 2641,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0037</text>\n",
       "</g>\n",
       "<!-- 140404120213392&#45;&gt;140404120269136* -->\n",
       "<g id=\"edge200\" class=\"edge\">\n",
       "<title>140404120213392&#45;&gt;140404120269136*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2727.15,-568.5C2736.09,-568.5 2744.73,-568.5 2752.57,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2752.73,-572 2762.73,-568.5 2752.73,-565 2752.73,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120213456 -->\n",
       "<g id=\"node155\" class=\"node\">\n",
       "<title>140404120213456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1902,-880.5 1902,-916.5 2093,-916.5 2093,-880.5 1902,-880.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1912\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1922,-880.5 1922,-916.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1964.5\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.4937</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2007,-880.5 2007,-916.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-894.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0111</text>\n",
       "</g>\n",
       "<!-- 140404120213456&#45;&gt;140404120259600+ -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>140404120213456&#45;&gt;140404120259600+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2063.15,-916.51C2073.21,-919.64 2083.43,-923.01 2093,-926.5 2103.26,-930.24 2114.25,-934.82 2124.08,-939.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2122.9,-942.45 2133.46,-943.34 2125.76,-936.06 2122.9,-942.45\"/>\n",
       "</g>\n",
       "<!-- 140404120213584 -->\n",
       "<g id=\"node156\" class=\"node\">\n",
       "<title>140404120213584</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1585,-358.5 1585,-394.5 1776,-394.5 1776,-358.5 1585,-358.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1595\" y=\"-372.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1605,-358.5 1605,-394.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1647.5\" y=\"-372.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9488</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1690,-358.5 1690,-394.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1733\" y=\"-372.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0221</text>\n",
       "</g>\n",
       "<!-- 140404120213584&#45;&gt;140404120269904* -->\n",
       "<g id=\"edge145\" class=\"edge\">\n",
       "<title>140404120213584&#45;&gt;140404120269904*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1776.15,-376.5C1785.09,-376.5 1793.73,-376.5 1801.57,-376.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.73,-380 1811.73,-376.5 1801.73,-373 1801.73,-380\"/>\n",
       "</g>\n",
       "<!-- 140404120270928 -->\n",
       "<g id=\"node157\" class=\"node\">\n",
       "<title>140404120270928</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2538,-440.5 2538,-476.5 2725,-476.5 2725,-440.5 2538,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2548\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2558,-440.5 2558,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2600.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.6796</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2643,-440.5 2643,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120270928&#45;&gt;140404120271184+ -->\n",
       "<g id=\"edge104\" class=\"edge\">\n",
       "<title>140404120270928&#45;&gt;140404120271184+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2709.26,-440.44C2715.51,-437.59 2721.51,-434.3 2727,-430.5 2747.74,-416.14 2764.48,-392.63 2775.31,-374.46\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2778.5,-375.94 2780.43,-365.52 2772.42,-372.46 2778.5,-375.94\"/>\n",
       "</g>\n",
       "<!-- 140404120270928*&#45;&gt;140404120270928 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>140404120270928*&#45;&gt;140404120270928</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2495.2,-503C2507.03,-497.37 2522.14,-490.58 2536,-485.5 2541.49,-483.49 2547.21,-481.53 2552.99,-479.65\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2554.27,-482.91 2562.75,-476.56 2552.16,-476.24 2554.27,-482.91\"/>\n",
       "</g>\n",
       "<!-- 140404120213648 -->\n",
       "<g id=\"node159\" class=\"node\">\n",
       "<title>140404120213648</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904,-330.5 1904,-366.5 2091,-366.5 2091,-330.5 1904,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924,-330.5 1924,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.3621</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2009,-330.5 2009,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0180</text>\n",
       "</g>\n",
       "<!-- 140404120213648&#45;&gt;140404120270416* -->\n",
       "<g id=\"edge105\" class=\"edge\">\n",
       "<title>140404120213648&#45;&gt;140404120270416*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2091.35,-348.5C2100.97,-348.5 2110.29,-348.5 2118.7,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2118.81,-352 2128.81,-348.5 2118.81,-345 2118.81,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120148176 -->\n",
       "<g id=\"node160\" class=\"node\">\n",
       "<title>140404120148176</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953,-550.5 953,-586.5 1140,-586.5 1140,-550.5 953,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973,-550.5 973,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.1116</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1058,-550.5 1058,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120148176&#45;&gt;140404120153040+ -->\n",
       "<g id=\"edge173\" class=\"edge\">\n",
       "<title>140404120148176&#45;&gt;140404120153040+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1112.15,-550.49C1122.21,-547.36 1132.43,-543.99 1142,-540.5 1152.26,-536.76 1163.25,-532.18 1173.08,-527.85\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1174.76,-530.94 1182.46,-523.66 1171.9,-524.55 1174.76,-530.94\"/>\n",
       "</g>\n",
       "<!-- 140404120148176+&#45;&gt;140404120148176 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>140404120148176+&#45;&gt;140404120148176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M915.19,-568.5C923.22,-568.5 932.59,-568.5 942.55,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"942.68,-572 952.68,-568.5 942.68,-565 942.68,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120213712 -->\n",
       "<g id=\"node162\" class=\"node\">\n",
       "<title>140404120213712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2221.5,-495.5 2221.5,-531.5 2407.5,-531.5 2407.5,-495.5 2221.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2231.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2241.5,-495.5 2241.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2281.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7171</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2321.5,-495.5 2321.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2364.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0222</text>\n",
       "</g>\n",
       "<!-- 140404120213712&#45;&gt;140404120270928* -->\n",
       "<g id=\"edge184\" class=\"edge\">\n",
       "<title>140404120213712&#45;&gt;140404120270928*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2407.89,-513.5C2417.67,-513.5 2427.13,-513.5 2435.66,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2435.91,-517 2445.91,-513.5 2435.91,-510 2435.91,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120213776 -->\n",
       "<g id=\"node163\" class=\"node\">\n",
       "<title>140404120213776</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2538,-165.5 2538,-201.5 2725,-201.5 2725,-165.5 2538,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2548\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2558,-165.5 2558,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2600.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.3840</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2643,-165.5 2643,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2684\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0077</text>\n",
       "</g>\n",
       "<!-- 140404120213776&#45;&gt;140404120271440* -->\n",
       "<g id=\"edge125\" class=\"edge\">\n",
       "<title>140404120213776&#45;&gt;140404120271440*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2700.25,-201.56C2709.31,-204.35 2718.43,-207.36 2727,-210.5 2737.5,-214.35 2748.72,-219.18 2758.69,-223.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2757.26,-226.94 2767.8,-228 2760.22,-220.6 2757.26,-226.94\"/>\n",
       "</g>\n",
       "<!-- 140404120271184 -->\n",
       "<g id=\"node164\" class=\"node\">\n",
       "<title>140404120271184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2855,-330.5 2855,-366.5 3042,-366.5 3042,-330.5 2855,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2865\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2875,-330.5 2875,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2917.5\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.9165</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2960,-330.5 2960,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120271696+ -->\n",
       "<g id=\"node176\" class=\"node\">\n",
       "<title>140404120271696+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3107\" cy=\"-238.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3107\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120271184&#45;&gt;140404120271696+ -->\n",
       "<g id=\"edge88\" class=\"edge\">\n",
       "<title>140404120271184&#45;&gt;140404120271696+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3026.26,-330.44C3032.51,-327.59 3038.51,-324.3 3044,-320.5 3064.74,-306.14 3081.48,-282.63 3092.31,-264.46\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3095.5,-265.94 3097.43,-255.52 3089.42,-262.46 3095.5,-265.94\"/>\n",
       "</g>\n",
       "<!-- 140404120271184+&#45;&gt;140404120271184 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>140404120271184+&#45;&gt;140404120271184</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-348.5C2825.22,-348.5 2834.59,-348.5 2844.55,-348.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2844.68,-352 2854.68,-348.5 2844.68,-345 2844.68,-352\"/>\n",
       "</g>\n",
       "<!-- 140404120213840 -->\n",
       "<g id=\"node166\" class=\"node\">\n",
       "<title>140404120213840</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1904,-220.5 1904,-256.5 2091,-256.5 2091,-220.5 1904,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1914\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1924,-220.5 1924,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8544</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2009,-220.5 2009,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2050\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120213840&#45;&gt;140404120270160+ -->\n",
       "<g id=\"edge169\" class=\"edge\">\n",
       "<title>140404120213840&#45;&gt;140404120270160+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2066.25,-256.56C2075.31,-259.35 2084.43,-262.36 2093,-265.5 2103.5,-269.35 2114.72,-274.18 2124.69,-278.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2123.26,-281.94 2133.8,-283 2126.22,-275.6 2123.26,-281.94\"/>\n",
       "</g>\n",
       "<!-- 140404120214032 -->\n",
       "<g id=\"node167\" class=\"node\">\n",
       "<title>140404120214032</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-330.5 3487,-366.5 3678,-366.5 3678,-330.5 3487,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-330.5 3507,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.4377</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-330.5 3592,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0414</text>\n",
       "</g>\n",
       "<!-- 140404120214032&#45;&gt;140404120272208* -->\n",
       "<g id=\"edge181\" class=\"edge\">\n",
       "<title>140404120214032&#45;&gt;140404120272208*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3651.25,-366.56C3660.31,-369.35 3669.43,-372.36 3678,-375.5 3688.5,-379.35 3699.72,-384.18 3709.69,-388.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3708.26,-391.94 3718.8,-393 3711.22,-385.6 3708.26,-391.94\"/>\n",
       "</g>\n",
       "<!-- 140404120214096 -->\n",
       "<g id=\"node168\" class=\"node\">\n",
       "<title>140404120214096</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-440.5 3487,-476.5 3678,-476.5 3678,-440.5 3487,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-440.5 3507,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.8122</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-440.5 3592,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0634</text>\n",
       "</g>\n",
       "<!-- 140404120214096&#45;&gt;140404120272720* -->\n",
       "<g id=\"edge141\" class=\"edge\">\n",
       "<title>140404120214096&#45;&gt;140404120272720*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3678.15,-458.5C3687.09,-458.5 3695.73,-458.5 3703.57,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3703.73,-462 3713.73,-458.5 3703.73,-455 3703.73,-462\"/>\n",
       "</g>\n",
       "<!-- 140404120271440 -->\n",
       "<g id=\"node169\" class=\"node\">\n",
       "<title>140404120271440</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2855,-220.5 2855,-256.5 3042,-256.5 3042,-220.5 2855,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2865\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2875,-220.5 2875,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2917.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.1271</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2960,-220.5 2960,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3001\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120271440&#45;&gt;140404120271696+ -->\n",
       "<g id=\"edge84\" class=\"edge\">\n",
       "<title>140404120271440&#45;&gt;140404120271696+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3042.35,-238.5C3051.97,-238.5 3061.29,-238.5 3069.7,-238.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3069.81,-242 3079.81,-238.5 3069.81,-235 3069.81,-242\"/>\n",
       "</g>\n",
       "<!-- 140404120271440*&#45;&gt;140404120271440 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>140404120271440*&#45;&gt;140404120271440</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2817.19,-238.5C2825.22,-238.5 2834.59,-238.5 2844.55,-238.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2844.68,-242 2854.68,-238.5 2844.68,-235 2844.68,-242\"/>\n",
       "</g>\n",
       "<!-- 140404120148624 -->\n",
       "<g id=\"node171\" class=\"node\">\n",
       "<title>140404120148624</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"638.5,-440.5 638.5,-476.5 820.5,-476.5 820.5,-440.5 638.5,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"648.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"658.5,-440.5 658.5,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"738.5,-440.5 738.5,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0009</text>\n",
       "</g>\n",
       "<!-- 140404120148624&#45;&gt;140404120148880* -->\n",
       "<g id=\"edge159\" class=\"edge\">\n",
       "<title>140404120148624&#45;&gt;140404120148880*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M798.25,-476.56C807.31,-479.35 816.43,-482.36 825,-485.5 835.5,-489.35 846.72,-494.18 856.69,-498.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"855.26,-501.94 865.8,-503 858.22,-495.6 855.26,-501.94\"/>\n",
       "</g>\n",
       "<!-- 140404120214160 -->\n",
       "<g id=\"node172\" class=\"node\">\n",
       "<title>140404120214160</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3487,-550.5 3487,-586.5 3678,-586.5 3678,-550.5 3487,-550.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3497\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3507,-550.5 3507,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7850</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3592,-550.5 3592,-586.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-564.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0683</text>\n",
       "</g>\n",
       "<!-- 140404120214160&#45;&gt;140404120285584* -->\n",
       "<g id=\"edge167\" class=\"edge\">\n",
       "<title>140404120214160&#45;&gt;140404120285584*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3678.15,-568.5C3687.09,-568.5 3695.73,-568.5 3703.57,-568.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3703.73,-572 3713.73,-568.5 3703.73,-565 3703.73,-572\"/>\n",
       "</g>\n",
       "<!-- 140404120214224 -->\n",
       "<g id=\"node173\" class=\"node\">\n",
       "<title>140404120214224</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3489.5,-275.5 3489.5,-311.5 3675.5,-311.5 3675.5,-275.5 3489.5,-275.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3499.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3509.5,-275.5 3509.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3549.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7867</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3589.5,-275.5 3589.5,-311.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3632.5\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0591</text>\n",
       "</g>\n",
       "<!-- 140404120214224&#45;&gt;140404120286096* -->\n",
       "<g id=\"edge89\" class=\"edge\">\n",
       "<title>140404120214224&#45;&gt;140404120286096*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3675.89,-293.5C3685.67,-293.5 3695.13,-293.5 3703.66,-293.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3703.91,-297 3713.91,-293.5 3703.91,-290 3703.91,-297\"/>\n",
       "</g>\n",
       "<!-- 140404120214288 -->\n",
       "<g id=\"node174\" class=\"node\">\n",
       "<title>140404120214288</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3804,-330.5 3804,-366.5 3986,-366.5 3986,-330.5 3804,-330.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3814\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3824,-330.5 3824,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3864\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.9509</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3904,-330.5 3904,-366.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3945\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120272464+ -->\n",
       "<g id=\"node194\" class=\"node\">\n",
       "<title>140404120272464+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"4115.5\" cy=\"-403.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"4115.5\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120214288&#45;&gt;140404120272464+ -->\n",
       "<g id=\"edge95\" class=\"edge\">\n",
       "<title>140404120214288&#45;&gt;140404120272464+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3967.55,-366.51C4005.29,-376.01 4049.87,-387.23 4079.96,-394.8\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4079.4,-398.27 4089.96,-397.32 4081.11,-391.49 4079.4,-398.27\"/>\n",
       "</g>\n",
       "<!-- 140404120271696 -->\n",
       "<g id=\"node175\" class=\"node\">\n",
       "<title>140404120271696</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3172,-220.5 3172,-256.5 3359,-256.5 3359,-220.5 3172,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3182\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3192,-220.5 3192,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3234.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0436</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3277,-220.5 3277,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3318\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0234</text>\n",
       "</g>\n",
       "<!-- 140404120271824tanh -->\n",
       "<g id=\"node183\" class=\"node\">\n",
       "<title>140404120271824tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"3424\" cy=\"-238.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"3424\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140404120271696&#45;&gt;140404120271824tanh -->\n",
       "<g id=\"edge208\" class=\"edge\">\n",
       "<title>140404120271696&#45;&gt;140404120271824tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3359.35,-238.5C3368.97,-238.5 3378.29,-238.5 3386.7,-238.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3386.81,-242 3396.81,-238.5 3386.81,-235 3386.81,-242\"/>\n",
       "</g>\n",
       "<!-- 140404120271696+&#45;&gt;140404120271696 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>140404120271696+&#45;&gt;140404120271696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3134.19,-238.5C3142.22,-238.5 3151.59,-238.5 3161.55,-238.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3161.68,-242 3171.68,-238.5 3161.68,-235 3161.68,-242\"/>\n",
       "</g>\n",
       "<!-- 140404120148880 -->\n",
       "<g id=\"node177\" class=\"node\">\n",
       "<title>140404120148880</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"955.5,-495.5 955.5,-531.5 1137.5,-531.5 1137.5,-495.5 955.5,-495.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"965.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"975.5,-495.5 975.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1015.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.9369</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1055.5,-495.5 1055.5,-531.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1096.5\" y=\"-509.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0010</text>\n",
       "</g>\n",
       "<!-- 140404120148880&#45;&gt;140404120153040+ -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>140404120148880&#45;&gt;140404120153040+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1137.63,-513.5C1148.18,-513.5 1158.43,-513.5 1167.6,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.82,-517 1177.82,-513.5 1167.82,-510 1167.82,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120148880*&#45;&gt;140404120148880 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>140404120148880*&#45;&gt;140404120148880</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M915.19,-513.5C923.92,-513.5 934.23,-513.5 945.17,-513.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"945.36,-517 955.36,-513.5 945.36,-510 945.36,-517\"/>\n",
       "</g>\n",
       "<!-- 140404120214416 -->\n",
       "<g id=\"node179\" class=\"node\">\n",
       "<title>140404120214416</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4.5,-839.5 4.5,-875.5 186.5,-875.5 186.5,-839.5 4.5,-839.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"14.5\" y=\"-853.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"24.5,-839.5 24.5,-875.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"64.5\" y=\"-853.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"104.5,-839.5 104.5,-875.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"145.5\" y=\"-853.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0033</text>\n",
       "</g>\n",
       "<!-- 140404120214416&#45;&gt;140404120214480* -->\n",
       "<g id=\"edge153\" class=\"edge\">\n",
       "<title>140404120214416&#45;&gt;140404120214480*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M186.63,-873.05C197.59,-874.94 208.23,-876.78 217.67,-878.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.31,-881.89 227.76,-880.15 218.5,-875 217.31,-881.89\"/>\n",
       "</g>\n",
       "<!-- 140404120214480 -->\n",
       "<g id=\"node180\" class=\"node\">\n",
       "<title>140404120214480</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"317,-866.5 317,-902.5 508,-902.5 508,-866.5 317,-866.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"327\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"337,-866.5 337,-902.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"379.5\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.7547</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"422,-866.5 422,-902.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120214480&#45;&gt;140404120214736+ -->\n",
       "<g id=\"edge113\" class=\"edge\">\n",
       "<title>140404120214480&#45;&gt;140404120214736+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M508.15,-884.5C517.09,-884.5 525.73,-884.5 533.57,-884.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"533.73,-888 543.73,-884.5 533.73,-881 533.73,-888\"/>\n",
       "</g>\n",
       "<!-- 140404120214480*&#45;&gt;140404120214480 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>140404120214480*&#45;&gt;140404120214480</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M281.19,-884.5C288.67,-884.5 297.31,-884.5 306.51,-884.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.68,-888 316.68,-884.5 306.68,-881 306.68,-888\"/>\n",
       "</g>\n",
       "<!-- 140404120271824 -->\n",
       "<g id=\"node182\" class=\"node\">\n",
       "<title>140404120271824</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3489,-220.5 3489,-256.5 3676,-256.5 3676,-220.5 3489,-220.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3499\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3509,-220.5 3509,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3551.5\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;0.7793</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3594,-220.5 3594,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-234.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0596</text>\n",
       "</g>\n",
       "<!-- 140404120271824&#45;&gt;140404120286096* -->\n",
       "<g id=\"edge196\" class=\"edge\">\n",
       "<title>140404120271824&#45;&gt;140404120286096*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3651.25,-256.56C3660.31,-259.35 3669.43,-262.36 3678,-265.5 3688.5,-269.35 3699.72,-274.18 3709.69,-278.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3708.26,-281.94 3718.8,-283 3711.22,-275.6 3708.26,-281.94\"/>\n",
       "</g>\n",
       "<!-- 140404120271824tanh&#45;&gt;140404120271824 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>140404120271824tanh&#45;&gt;140404120271824</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3451.19,-238.5C3459.22,-238.5 3468.59,-238.5 3478.55,-238.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3478.68,-242 3488.68,-238.5 3478.68,-235 3478.68,-242\"/>\n",
       "</g>\n",
       "<!-- 140404120214736 -->\n",
       "<g id=\"node184\" class=\"node\">\n",
       "<title>140404120214736</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"634,-866.5 634,-902.5 825,-902.5 825,-866.5 634,-866.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"644\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"654,-866.5 654,-902.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"696.5\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;2.2983</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-866.5 739,-902.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"782\" y=\"-880.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120215312+ -->\n",
       "<g id=\"node196\" class=\"node\">\n",
       "<title>140404120215312+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"888\" cy=\"-870.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-866.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140404120214736&#45;&gt;140404120215312+ -->\n",
       "<g id=\"edge129\" class=\"edge\">\n",
       "<title>140404120214736&#45;&gt;140404120215312+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M825.15,-876.03C834.28,-875.22 843.1,-874.43 851.08,-873.71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"851.39,-877.2 861.04,-872.82 850.76,-870.23 851.39,-877.2\"/>\n",
       "</g>\n",
       "<!-- 140404120214736+&#45;&gt;140404120214736 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>140404120214736+&#45;&gt;140404120214736</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.19,-884.5C605.67,-884.5 614.31,-884.5 623.51,-884.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.68,-888 633.68,-884.5 623.68,-881 623.68,-888\"/>\n",
       "</g>\n",
       "<!-- 140404120001744 -->\n",
       "<g id=\"node186\" class=\"node\">\n",
       "<title>140404120001744</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1270,-55.5 1270,-91.5 1457,-91.5 1457,-55.5 1270,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1280\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1290,-55.5 1290,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1332.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.8433</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1375,-55.5 1375,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0068</text>\n",
       "</g>\n",
       "<!-- 140404120001744&#45;&gt;140404045641552+ -->\n",
       "<g id=\"edge202\" class=\"edge\">\n",
       "<title>140404120001744&#45;&gt;140404045641552+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1457.35,-73.5C1466.97,-73.5 1476.29,-73.5 1484.7,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1484.81,-77 1494.81,-73.5 1484.81,-70 1484.81,-77\"/>\n",
       "</g>\n",
       "<!-- 140404120001744*&#45;&gt;140404120001744 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>140404120001744*&#45;&gt;140404120001744</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.19,-73.5C1240.22,-73.5 1249.59,-73.5 1259.55,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1259.68,-77 1269.68,-73.5 1259.68,-70 1259.68,-77\"/>\n",
       "</g>\n",
       "<!-- 140404120272208 -->\n",
       "<g id=\"node188\" class=\"node\">\n",
       "<title>140404120272208</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3804,-385.5 3804,-421.5 3986,-421.5 3986,-385.5 3804,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"3814\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3824,-385.5 3824,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3864\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.2390</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"3904,-385.5 3904,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"3945\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120272208&#45;&gt;140404120272464+ -->\n",
       "<g id=\"edge174\" class=\"edge\">\n",
       "<title>140404120272208&#45;&gt;140404120272464+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3986.13,-403.5C4018.43,-403.5 4053.03,-403.5 4078.16,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4078.32,-407 4088.32,-403.5 4078.32,-400 4078.32,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120272208*&#45;&gt;140404120272208 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>140404120272208*&#45;&gt;140404120272208</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3768.13,-403.5C3775.67,-403.5 3784.39,-403.5 3793.64,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3793.87,-407 3803.87,-403.5 3793.87,-400 3793.87,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120214992 -->\n",
       "<g id=\"node190\" class=\"node\">\n",
       "<title>140404120214992</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"319.5,-756.5 319.5,-792.5 505.5,-792.5 505.5,-756.5 319.5,-756.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"339.5,-756.5 339.5,-792.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"379.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"419.5,-756.5 419.5,-792.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"462.5\" y=\"-770.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0008</text>\n",
       "</g>\n",
       "<!-- 140404120214992&#45;&gt;140404120215056* -->\n",
       "<g id=\"edge102\" class=\"edge\">\n",
       "<title>140404120214992&#45;&gt;140404120215056*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M481.25,-792.56C490.31,-795.35 499.43,-798.36 508,-801.5 518.5,-805.35 529.72,-810.18 539.69,-814.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"538.26,-817.94 548.8,-819 541.22,-811.6 538.26,-817.94\"/>\n",
       "</g>\n",
       "<!-- 140404120215056 -->\n",
       "<g id=\"node191\" class=\"node\">\n",
       "<title>140404120215056</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"636.5,-811.5 636.5,-847.5 822.5,-847.5 822.5,-811.5 636.5,-811.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"646.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"656.5,-811.5 656.5,-847.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"696.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6556</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"736.5,-811.5 736.5,-847.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-825.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120215056&#45;&gt;140404120215312+ -->\n",
       "<g id=\"edge101\" class=\"edge\">\n",
       "<title>140404120215056&#45;&gt;140404120215312+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M799.6,-847.6C818,-852.42 837.09,-857.43 852.72,-861.52\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"851.88,-864.92 862.44,-864.07 853.66,-858.15 851.88,-864.92\"/>\n",
       "</g>\n",
       "<!-- 140404120215056*&#45;&gt;140404120215056 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>140404120215056*&#45;&gt;140404120215056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.19,-829.5C606.35,-829.5 615.89,-829.5 626.03,-829.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"626.35,-833 636.35,-829.5 626.35,-826 626.35,-833\"/>\n",
       "</g>\n",
       "<!-- 140404120272464 -->\n",
       "<g id=\"node193\" class=\"node\">\n",
       "<title>140404120272464</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4245,-385.5 4245,-421.5 4427,-421.5 4427,-385.5 4245,-385.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4255\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4265,-385.5 4265,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4305\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.1899</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4345,-385.5 4345,-421.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4386\" y=\"-399.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120272464&#45;&gt;140404120285328+ -->\n",
       "<g id=\"edge108\" class=\"edge\">\n",
       "<title>140404120272464&#45;&gt;140404120285328+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4427.04,-414.75C4458.84,-418.73 4492.79,-422.97 4517.41,-426.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4517.01,-429.53 4527.36,-427.3 4517.88,-422.58 4517.01,-429.53\"/>\n",
       "</g>\n",
       "<!-- 140404120272464+&#45;&gt;140404120272464 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>140404120272464+&#45;&gt;140404120272464</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4142.86,-403.5C4165.79,-403.5 4200.56,-403.5 4234.24,-403.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4234.65,-407 4244.65,-403.5 4234.65,-400 4234.65,-407\"/>\n",
       "</g>\n",
       "<!-- 140404120215312 -->\n",
       "<g id=\"node195\" class=\"node\">\n",
       "<title>140404120215312</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"951,-852.5 951,-888.5 1142,-888.5 1142,-852.5 951,-852.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"961\" y=\"-866.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"971,-852.5 971,-888.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-866.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.6427</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1056,-852.5 1056,-888.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1099\" y=\"-866.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0038</text>\n",
       "</g>\n",
       "<!-- 140404120215312&#45;&gt;140404120240528+ -->\n",
       "<g id=\"edge209\" class=\"edge\">\n",
       "<title>140404120215312&#45;&gt;140404120240528+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.15,-870.5C1151.09,-870.5 1159.73,-870.5 1167.57,-870.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.73,-874 1177.73,-870.5 1167.73,-867 1167.73,-874\"/>\n",
       "</g>\n",
       "<!-- 140404120215312+&#45;&gt;140404120215312 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>140404120215312+&#45;&gt;140404120215312</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M915.19,-870.5C922.67,-870.5 931.31,-870.5 940.51,-870.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"940.68,-874 950.68,-870.5 940.68,-867 940.68,-874\"/>\n",
       "</g>\n",
       "<!-- 140404120272720 -->\n",
       "<g id=\"node197\" class=\"node\">\n",
       "<title>140404120272720</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4245,-440.5 4245,-476.5 4427,-476.5 4427,-440.5 4245,-440.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"4255\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4265,-440.5 4265,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4305\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.6791</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"4345,-440.5 4345,-476.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"4386\" y=\"-454.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0758</text>\n",
       "</g>\n",
       "<!-- 140404120272720&#45;&gt;140404120285328+ -->\n",
       "<g id=\"edge149\" class=\"edge\">\n",
       "<title>140404120272720&#45;&gt;140404120285328+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4427.04,-446.83C4458.84,-442.71 4492.79,-438.31 4517.41,-435.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4517.9,-438.58 4527.36,-433.82 4517,-431.64 4517.9,-438.58\"/>\n",
       "</g>\n",
       "<!-- 140404120272720*&#45;&gt;140404120272720 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>140404120272720*&#45;&gt;140404120272720</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3768.25,-458.5C3848.61,-458.5 4094.33,-458.5 4234.42,-458.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4234.74,-462 4244.74,-458.5 4234.74,-455 4234.74,-462\"/>\n",
       "</g>\n",
       "<!-- 140404120002448 -->\n",
       "<g id=\"node199\" class=\"node\">\n",
       "<title>140404120002448</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"953.5,-28.5 953.5,-64.5 1139.5,-64.5 1139.5,-28.5 953.5,-28.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"963.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"973.5,-28.5 973.5,-64.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1053.5,-28.5 1053.5,-64.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1096.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.0063</text>\n",
       "</g>\n",
       "<!-- 140404120002448&#45;&gt;140404120001744* -->\n",
       "<g id=\"edge118\" class=\"edge\">\n",
       "<title>140404120002448&#45;&gt;140404120001744*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1139.89,-62.44C1149.96,-64.18 1159.71,-65.86 1168.43,-67.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1168.07,-70.85 1178.52,-69.1 1169.26,-63.96 1168.07,-70.85\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tiny Dataset with loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "    [2.0, 3.0, -1.0],\n",
    "    [3.0, -1.0, 0.5],\n",
    "    [0.5, 1.0, 1.0], \n",
    "    [1.0, 1.0, -1.0]\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.145450264034548</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m6\u001b[0m\u001b[1;36m.145450264034548\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ypred = [n(x) for x in xs]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9613622740038076</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5091506010451603</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9497390552324829</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7451677610062515</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9613622740038076\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.5091506010451603\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9497390552324829\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7451677610062515\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.04473465638916681</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-0.04473465638916681\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n.layers[0].neurons[0].w[0].grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8773320545613115</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-0.8773320545613115\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n.layers[0].neurons[0].w[0].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in n.parameters():\n",
    "    p.data += -0.01 * p.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8768847079974198</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m-0.8768847079974198\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n.layers[0].neurons[0].w[0].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5.9448254035134465</span><span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m5\u001b[0m\u001b[1;36m.9448254035134465\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ypred = [n(x) for x in xs]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9617265322081271</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.43565539138090165</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9510677256835314</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.725065694807515</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9617265322081271\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.43565539138090165\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9510677256835314\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.725065694807515\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8768847079974198</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.21949508647796803</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1385938942733024</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5427509639370535</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4989987951971535</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9810637753982486</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8032841357361882</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.10822461638644491</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2191073445250642</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8693341479297328</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5150553447581354</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1599712708867693</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.9204522149327646</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6176978033162941</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.33248584468999454</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6728960760418363</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7467247012385713</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6928044491564865</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3733392595509354</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.03928302778692913</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8017417247095591</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.14178985429307983</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.9916730215251183</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.23454886497577554</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5461943160052498</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.09193046790217876</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.6258347045651111</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.768590750645844</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8076512515956322</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.6702462673395139</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.49429632745656293</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.9460952730831453</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3596651251112354</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7141165005167276</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3806245013086102</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8570897419717581</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.41643954968132757</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8282591500009012</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8021962015254895</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7707916705043153</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9262723943108128</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8768847079974198\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.21949508647796803\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.1385938942733024\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5427509639370535\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.4989987951971535\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9810637753982486\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8032841357361882\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.10822461638644491\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.2191073445250642\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8693341479297328\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5150553447581354\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.1599712708867693\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.9204522149327646\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6176978033162941\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.33248584468999454\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6728960760418363\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7467247012385713\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.6928044491564865\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.3733392595509354\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.03928302778692913\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8017417247095591\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.14178985429307983\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.9916730215251183\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.23454886497577554\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.5461943160052498\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.09193046790217876\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.6258347045651111\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.768590750645844\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8076512515956322\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.6702462673395139\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.49429632745656293\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.9460952730831453\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.3596651251112354\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7141165005167276\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.3806245013086102\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8570897419717581\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.41643954968132757\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8282591500009012\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8021962015254895\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.7707916705043153\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9262723943108128\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n.parameters()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make the above `Repeat` section into `training` loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(repeats, model, xs, ygt, lr = 0.01):\n",
    "    \n",
    "    for k in range(repeats):\n",
    "        # forward pass\n",
    "        ypred = [model(x) for x in xs]\n",
    "        loss = sum((yout - ygt)**2 for ygt, yout in zip(ygt, ypred))\n",
    "        print(k, loss.data)\n",
    "        \n",
    "        # backward propagation\n",
    "        for p in n.parameters(): p.grad = 0.0 # zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        # update: gradient descent\n",
    "        for p in model.parameters(): p.data += -lr * p.grad\n",
    "    return ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "    [2.0, 3.0, -1.0],\n",
    "    [3.0, -1.0, 0.5],\n",
    "    [0.5, 1.0, 1.0], \n",
    "    [1.0, 1.0, -1.0]\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(3, [4, 4, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7.220472885146723</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m0\u001b[0m \u001b[1;36m7.220472885146723\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.8361918435934355</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m1\u001b[0m \u001b[1;36m6.8361918435934355\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5.589046560434221</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m2\u001b[0m \u001b[1;36m5.589046560434221\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.147937671605216</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m3\u001b[0m \u001b[1;36m4.147937671605216\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.875203300452224</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m4\u001b[0m \u001b[1;36m4.875203300452224\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4.323610795464504</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m5\u001b[0m \u001b[1;36m4.323610795464504\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.546174609092321</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m6\u001b[0m \u001b[1;36m3.546174609092321\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.7412962395817813</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m7\u001b[0m \u001b[1;36m1.7412962395817813\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5351626826196177</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m8\u001b[0m \u001b[1;36m0.5351626826196177\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.21335050879201323</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m9\u001b[0m \u001b[1;36m0.21335050879201323\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5781623601547444</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8870687314798208</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8905626911927673</span><span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Value</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">label</span>= <span style=\"color: #808000; text-decoration-color: #808000\">data</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.896687278453952</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.5781623601547444\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8870687314798208\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m-0.8905626911927673\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mValue\u001b[0m\u001b[1m(\u001b[0m\u001b[33mlabel\u001b[0m= \u001b[33mdata\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.896687278453952\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(10, model, xs, ys, 0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Done!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "zhnn",
   "language": "python",
   "name": "zhnn"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}