nbs/exercises/04_building_makemore_mlp2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "35220ee2",
   "metadata": {},
   "source": [
    "# Building Makemore MLP 2 Exercise"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81bb419a",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a95f9510",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import numpy\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plot\n",
    "import random\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c53b85f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "g = torch.Generator().manual_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92ac3ed2",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f1d2011",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = open('../data/names.txt', 'r').read().splitlines()\n",
    "words[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b999c2a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set(''.join(words))))\n",
    "stoi = {s: i+1 for i, s in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i:s for s, i in stoi.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d2efc63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32033"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c49fe0ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_training_set(words, block_size, print_disabled=False):\n",
    "    \n",
    "    chars = sorted(list(set(''.join(words))))\n",
    "    stoi = {s: i+1 for i, s in enumerate(chars)}\n",
    "    stoi['.'] = 0\n",
    "    itos = {i:s for s, i in stoi.items()}\n",
    "    \n",
    "    X, Y = [], []\n",
    "    \n",
    "    for w in words:\n",
    "        if print_disabled: print(w)\n",
    "        \n",
    "        context = [0] * block_size\n",
    "        for ch in w + '.':\n",
    "            ix = stoi[ch]\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "            if print_disabled: print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "            context = context[1:] + [ix] # crop and append\n",
    "            \n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbc49697",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = generate_training_set(words, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c29afe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([228146, 3]), torch.Size([228146]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b8efd11",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_train_valid_test_split(words, block_size=3):\n",
    "    random.seed(42)\n",
    "    random.shuffle(words)\n",
    "    n1 = int(0.8*len(words))\n",
    "    n2 = int(0.9*len(words))\n",
    "\n",
    "    Xtr, Ytr = generate_training_set(words[:n1], block_size)\n",
    "    Xdev, Ydev = generate_training_set(words[n1:n2], block_size)\n",
    "    Xte, Yte = generate_training_set(words[n2:], block_size)\n",
    "    \n",
    "    return Xtr, Ytr, Xdev, Ydev, Xte, Yte"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cc4d38c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Ytr, Xdev, Ydev, Xte, Yte = generate_train_valid_test_split(words, block_size=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3597d6af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([182625, 3]), torch.Size([182625]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtr.shape, Ytr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2f38b13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([22655, 3]), torch.Size([22655]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xdev.shape, Ydev.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c88462e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([22866, 3]), torch.Size([22866]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xte.shape, Yte.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de45bb6",
   "metadata": {},
   "source": [
    "## E01\n",
    ">  I did not get around to seeing what happens when you initialize all weights and biases to zero. Try this and train the neural net. You might think either that \n",
    "> 1) the network trains just fine or \n",
    "> 2) the network doesn't train at all, but actually it is \n",
    "> 3) the network trains but only partially, and achieves a pretty bad final performance. \n",
    "\n",
    "> Inspect the gradients and activations to figure out what is happening and why the network is only partially training, and what part is being trained exactly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c91a5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X, Y, epochs, bs = 32, context_length = 3, vocab_size = 27, embedding_size = 10, n_hidden = 100, print_at_every_epoch=10):\n",
    "    \n",
    "    C = torch.randn((vocab_size, embedding_size), generator=g)\n",
    "    \n",
    "    layers = [\n",
    "        nn.Linear(context_length * embedding_size, n_hidden), nn.BatchNorm1d(n_hidden), nn.Tanh(),\n",
    "        nn.Linear(n_hidden, n_hidden), nn.BatchNorm1d(n_hidden), nn.Tanh(),\n",
    "        nn.Linear(n_hidden, n_hidden), nn.BatchNorm1d(n_hidden), nn.Tanh(),\n",
    "        nn.Linear(n_hidden, n_hidden), nn.BatchNorm1d(n_hidden), nn.Tanh(),\n",
    "        nn.Linear(n_hidden, n_hidden), nn.BatchNorm1d(n_hidden), nn.Tanh(),\n",
    "        nn.Linear(n_hidden, vocab_size), nn.BatchNorm1d(vocab_size)\n",
    "    ]\n",
    "    visualisation = [-1] * len(layers) \n",
    "\n",
    "    def get_activation(layer_num):\n",
    "        def hook_fn(m, i, o): visualisation[layer_num] = o \n",
    "        return hook_fn\n",
    "        \n",
    "    for i, layer in enumerate(layers):\n",
    "        layer.register_forward_hook(get_activation(i))\n",
    "        if isinstance(layer, nn.Linear):\n",
    "            nn.init.zeros_(layer.weight)\n",
    "            nn.init.zeros_(layer.bias)\n",
    "\n",
    "    print('Total number of layers are : ', len(layers))\n",
    "    parameters = [C] + [p for layer in layers for p in layer.parameters()]\n",
    "    print('Total number of parameters are :', sum(p.nelement() for p in parameters))\n",
    "\n",
    "    \n",
    "    # set parameters to required grad\n",
    "    for p in parameters:\n",
    "        p.requires_grad = True\n",
    "\n",
    "    \n",
    "        \n",
    "    lossi = []\n",
    "    ud = []\n",
    "    for i in range(epochs):\n",
    "        ix = torch.randint(0, len(X), (bs,),  generator=g)\n",
    "        \n",
    "        Xb, Yb = X[ix], Y[ix]\n",
    "        \n",
    "        emb = C[Xb]\n",
    "        x = emb.view(emb.shape[0], -1)\n",
    "        \n",
    "        for layer in layers:\n",
    "            x = layer(x)\n",
    "            x.retain_grad()\n",
    "\n",
    "\n",
    "        loss = F.cross_entropy(x, Yb)\n",
    "\n",
    "            \n",
    "        for p in parameters:\n",
    "            p.grad = None\n",
    "            \n",
    "        loss.backward()\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        lr = 0.1 if i <= 100_000 else 0.01 # lr decay\n",
    "        \n",
    "        for p in parameters:\n",
    "            p.data += -lr * p.grad\n",
    "        \n",
    "        if print_at_every_epoch: \n",
    "            if i % print_at_every_epoch == 0:\n",
    "                print(f'Epoch {i} Loss {loss.item()}')\n",
    "        \n",
    "        lossi.append(loss.item())\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            for p in parameters:\n",
    "                grad_update_data_ratio = (lr * p.grad.std() / p.data.std()).log10().item() \n",
    "                ud.append(grad_update_data_ratio)\n",
    "        \n",
    "    return C, layers, lossi, ud, visualisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ca0b450",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of layers are :  17\n",
      "Total number of parameters are : 47551\n",
      "Epoch 0 Loss 3.295837163925171\n",
      "Epoch 100 Loss 3.0274267196655273\n",
      "Epoch 200 Loss 2.801142930984497\n",
      "Epoch 300 Loss 2.839231491088867\n",
      "Epoch 400 Loss 2.77958083152771\n",
      "Epoch 500 Loss 2.7113969326019287\n",
      "Epoch 600 Loss 2.7721636295318604\n",
      "Epoch 700 Loss 2.801912784576416\n",
      "Epoch 800 Loss 2.7530055046081543\n",
      "Epoch 900 Loss 3.0189149379730225\n"
     ]
    }
   ],
   "source": [
    "C, layers, losses, uds, v = train(Xtr, Ytr, 1000, print_at_every_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baa5744f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[14].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6c20e92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[14].grad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4ed32e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_loss(model, embedding, X, Y):\n",
    "    emb = embedding[X]\n",
    "    logits = emb.view(emb.shape[0], -1)\n",
    "    for layer in model: logits = layer(logits) \n",
    "    return F.cross_entropy(logits, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c8bd001",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(2.8327, grad_fn=<NllLossBackward0>),\n",
       " tensor(2.8316, grad_fn=<NllLossBackward0>))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_loss(layers,C, Xtr, Ytr), evaluate_loss(layers,C, Xdev, Ydev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff20d952",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb = C[Xdev]\n",
    "logits = emb.view(emb.shape[0], -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2c6a003",
   "metadata": {},
   "source": [
    "> The loss is not good. The network partially trains "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98f26a4c",
   "metadata": {},
   "source": [
    "### Lets see the activations and gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0849420d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_histograms(layers, C, Xdev, otype='activations'):\n",
    "    plot.figure(figsize=(20, 4))\n",
    "    legends = []\n",
    "    emb = C[Xdev]\n",
    "    logits = emb.view(emb.shape[0], -1)\n",
    "    for i, layer in enumerate(layers[:-1]):\n",
    "        if isinstance(layer, nn.Tanh):\n",
    "            t = v[i]\n",
    "            if otype == 'gradients': t = v[i].grad\n",
    "            print('layer %d (%10s): mean %+f, std %e' % (i, layer.__class__.__name__, t.mean(), t.std()))\n",
    "            hy, hx = torch.histogram(t, density=True)\n",
    "            plot.plot(hx[:-1].detach(), hy.detach())\n",
    "            legends.append(f'layer {i} ({layer.__class__.__name__})')\n",
    "    plot.legend(legends)\n",
    "    plot.title('gradient distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06bf1b7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 2 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 5 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 8 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 11 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 14 (      Tanh): mean +0.000000, std 0.000000e+00\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABkYAAAF0CAYAAAB2XoXVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABsHElEQVR4nO3deXhU5d3/8c+ZyWRfhJBkArIEQRQFAVMQLJoqBCUulCogFRtBUHF50MeiaMGgLVhAAUWxWlkqFURRSnlQ4KeAKFYhAkJxqRoWgRiBkED2zJzfH5OMHQkhCTOZzOT9uq65cubMfe77ewLXwOST7zmGaZqmAAAAAAAAAAAAmgGLvwsAAAAAAAAAAABoLAQjAAAAAAAAAACg2SAYAQAAAAAAAAAAzQbBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJoNghEAAAAAAAAAANBsEIwAAAAAAAAAAIBmg2AEAAAAAAAAAAA0GwQjAAAAQDPVoUMHZWZmup9v3LhRhmFo48aNPl33hRde0KJFi856HsMwlJWV5X6+aNEiGYahvXv31mueadOmaeXKlfU6pqa10tLSdPHFF9drnjNZs2aNxzn+t5//+QEAAACoG4IRAAAAAJKkXr166eOPP1avXr18uo63gpGfy8jI0Mcff6zk5OR6HdeQYKSha9XXmjVrNHXq1Bpfe/vttzV58mSfrg8AAAAEoxB/FwAAAACg7kzTVGlpqSIiIrw+d2xsrC677DKvz9tYEhISlJCQ4NM1SkpKFB4e3ihrnUnPnj39uj4AAAAQqOgYAQAAAPzgH//4h7p3766wsDB17NhRc+fOVVZWlgzD8BhnGIbuvfdevfjii7rwwgsVFhamxYsXS5KmTp2qPn36qGXLloqNjVWvXr30yiuvyDRNjzkqKio0ceJE2e12RUZG6pe//KU+/fTTU2o63aW0tm3bphtuuEEtW7ZUeHi4evbsqeXLl3uMqb601IYNG3T33XerVatWio+P19ChQ3Xo0CH3uA4dOujf//63Nm3aJMMwZBiGOnToUOv3qrCwUGPHjlV8fLyio6N1zTXX6Ouvvz5lXE2Xt9q+fbuuu+46JSYmKiwsTK1bt1ZGRoa+//579/e3qKhIixcvdteTlpbmMd+6des0evRoJSQkKDIyUmVlZbVetmvz5s267LLLFBERoTZt2mjy5MlyOBxn/D7v3btXhmG4u2kyMzP1/PPPu+usflSvWdOltPbv369bb73Vfb4XXnihnn76aTmdzlPWmTVrlp555hmlpKQoOjpaffv21b/+9a9a/ywAAACAYEDHCAAAANDI3n33XQ0dOlRXXHGFXn/9dVVWVmrWrFn64Ycfahy/cuVKbd68WVOmTJHdbldiYqIk1w+477zzTrVr106S9K9//Uv33XefDh48qClTpriPHzt2rP72t7/poYce0sCBA7V7924NHTpUJ06cOGOtGzZs0DXXXKM+ffroxRdfVFxcnJYtW6bhw4eruLj4lB/M33HHHcrIyNBrr72mAwcO6Pe//71uvfVWvf/++5Jcl3+66aabFBcXpxdeeEGSFBYWdtr1TdPUkCFDtGXLFk2ZMkW/+MUv9NFHH+naa689Y+1FRUUaOHCgUlJS9PzzzyspKUm5ubnasGGD+9w//vhjXXXVVfrVr37lvixVbGysxzyjR49WRkaGXn31VRUVFclms512zdzcXI0YMUKPPPKInnjiCf3f//2f/vjHPyo/P1/z5s07Y83/bfLkySoqKtKbb76pjz/+2L3/dJfv+vHHH9WvXz+Vl5frySefVIcOHbR69Wo99NBD+vbbb93f72rPP/+8LrjgAs2ZM8e93uDBg5WTk6O4uLh61QoAAAAEEoIRAAAAoJFNmTJFbdq00dq1axUaGipJuuaaa07bOXHy5Ent2rVLLVq08Ni/cOFC97bT6VRaWppM09TcuXM1efJkGYahL7/8UosXL9YDDzygGTNmSJIGDhyopKQk/fa3vz1jrePHj9dFF12k999/XyEhro8PgwYN0pEjR/Too4/qtttuk8XyUyP6Nddco2effdb9/NixY5o4caJyc3Nlt9vVs2dPRURE1PmyXWvXrtWGDRs0d+5c3X///e76Q0ND9dhjj9V67JdffqmjR4/qlVde0Y033ujeP2zYMPf2ZZddJovFooSEhNPWc/XVV+svf/nLGWuVpKNHj+of//iHbrjhBklSenq6SkpKNH/+fE2cONEdYtXFeeedp6SkJHedZ/LMM8/o4MGD+uSTT9S7d29Jrj8rh8OhF198URMmTND555/vHh8TE6PVq1fLarVKklq3bq3evXvrnXfe0YgRI+pcJwAAABBouJQWAAAA0IiKioq0bds2DRkyxB2KSFJ0dLSuv/76Go+56qqrTglFJOn999/XgAEDFBcXJ6vVKpvNpilTpujo0aPKy8uT5Or4kHRKCDJs2DB30HE633zzjb788kv3sZWVle7H4MGDdfjwYX311Vcex1QHAtW6d+8uSdq3b1+ta53O6eofOXLkGY/t1KmTWrRooYcfflgvvvii9uzZ06AafvOb39R5bExMzCnfg5EjR8rpdOqDDz5o0Pp19f7776tr167uUKRaZmamTNN0d+1Uy8jIcIci0tn/WQEAAACBgmAEAAAAaET5+fkyTdPdCfDfaton1XzppE8//VTp6emSpJdfflkfffSRtm7d6u6iKCkpkeTqYJAku93ucXxISIji4+NrrbX60l4PPfSQbDabx2P8+PGSpCNHjngc8/M5qy+TVV1PfR09erTGWn9+PjWJi4vTpk2b1KNHDz366KO66KKL1Lp1az3++OOqqKiocw2nu3RVTWr6M6yutfrPwleOHj1aY62tW7eucX1v/1kBAAAAgYJLaQEAAACNqEWLFjIMo8b7ieTm5tZ4zM9vyC5Jy5Ytk81m0+rVqxUeHu7ev3LlSo9x1T/8zs3NVZs2bdz7Kysrz/iD+latWkmSJk2apKFDh9Y4pkuXLrXOcbbi4+Pdtf73D/JP9736uW7dumnZsmUyTVOff/65Fi1apCeeeEIRERF65JFH6jRHTd//06ntz7W6/uo/r7KyMo9xPw+Z6is+Pl6HDx8+Zf+hQ4ck/fTnCQAAADR3dIwAAAAAjSgqKkqpqalauXKlysvL3ftPnjyp1atX13kewzAUEhLicSmkkpISvfrqqx7j0tLSJEl///vfPfYvX75clZWVta7RpUsXde7cWTt37lRqamqNj5iYmDrXXC0sLKzOXQm/+tWvaqz/tddeq9eahmHokksu0ezZs3XOOefos88+a1A9Z3LixAmtWrXqlFotFouuuOIKSXLfS+bzzz/3GPfz46prk+rWxXH11Vdrz549HucmSX/7299kGIb7ewkAAAA0d3SMAAAAAI3siSeeUEZGhgYNGqT/+Z//kcPh0MyZMxUdHa1jx47VaY6MjAw988wzGjlypMaNG6ejR49q1qxZ7h+kV7vwwgt16623as6cObLZbBowYIB2796tWbNmKTY29ozr/OUvf9G1116rQYMGKTMzU23atNGxY8f0xRdf6LPPPtMbb7xR7/Ov7uJ4/fXX1bFjR4WHh6tbt241jk1PT9cVV1yhiRMnqqioSKmpqfroo49OCYBqsnr1ar3wwgsaMmSIOnbsKNM09dZbb+n48eMaOHCgRz0bN27UP//5TyUnJysmJqbBnTDx8fG6++67tX//fp1//vlas2aNXn75Zd19993uG6/b7XYNGDBA06dPV4sWLdS+fXu99957euutt2r8XknSn//8Z1177bWyWq3q3r27x/1pqj3wwAP629/+poyMDD3xxBNq3769/u///k8vvPCC7r77bo8brwMAAADNGcEIAAAA0MiuueYarVixQlOmTNHw4cNlt9s1fvx4HTp0qE4/8JdcN2RfsGCB/vznP+v6669XmzZtNHbsWCUmJmrMmDEeY1955RUlJSVp0aJFevbZZ9WjRw+tWLFCI0aMOOM6v/rVr/Tpp5/qT3/6kyZMmKD8/HzFx8era9euGjZsWIPOf+rUqTp8+LDGjh2rEydOqH379tq7d2+NYy0Wi1atWqUHH3xQM2bMUHl5uS6//HKtWbNGF1xwQa3rdO7cWeecc45mzJihQ4cOKTQ0VF26dNGiRYv0u9/9zj1u7ty5uueeezRixAgVFxfryiuv1MaNGxt0bna7Xc8//7weeugh7dq1Sy1bttSjjz6qqVOneox79dVXdd999+nhhx+Ww+HQ9ddfr6VLlyo1NdVj3MiRI/XRRx/phRde0BNPPCHTNJWTk+PuOvlvCQkJ2rJliyZNmqRJkyapsLBQHTt21IwZM/Tggw826HwAAACAYGSYpmn6uwgAAACguauoqFCPHj3Upk0brVu3zt/lAAAAAEDQomMEAAAA8IMxY8Zo4MCBSk5OVm5url588UV98cUXmjt3rr9LAwAAAICgRjACAAAA+MGJEyf00EMP6ccff5TNZlOvXr20Zs0aDRgwwN+lAQAAAEBQ41JaAAAAAAAAAACg2bD4uwAAAAAAAAAAAIDGQjACAAAAAAAAAACaDYIRAAAAAAAAAADQbATkzdedTqcOHTqkmJgYGYbh73IAAAAAAAAAAIAfmaapEydOqHXr1rJYau8JCchg5NChQ2rbtq2/ywAAAAAAAAAAAE3IgQMHdO6559Y6JiCDkZiYGEmuE4yNjfVzNQAAAAAAAAAAwJ8KCwvVtm1bd35Qm4AMRqovnxUbG0swAgAAAAAAAAAAJKlOt9/g5usAAAAAAAAAAKDZIBgBAAAAAAAAAADNBsEIAAAAAAAAAABoNgLyHiMAAAAAAAAAgKbH6XSqvLzc32UgSIWGhspiOft+D4IRAAAAAAAAAMBZKy8vV05OjpxOp79LQZCyWCxKSUlRaGjoWc1DMAIAAAAAAAAAOCumaerw4cOyWq1q27atV36rH/hvTqdThw4d0uHDh9WuXTsZhtHguQhGAAAAAAAAAABnpbKyUsXFxWrdurUiIyP9XQ6CVEJCgg4dOqTKykrZbLYGz1Pv2O6DDz7Q9ddfr9atW8swDK1cudLjddM0lZWVpdatWysiIkJpaWn697//7TGmrKxM9913n1q1aqWoqCjdcMMN+v777xt8EgAAAAAAAAAA/3E4HJJ01pc4AmpT/fer+u9bQ9U7GCkqKtIll1yiefPm1fj6jBkz9Mwzz2jevHnaunWr7Ha7Bg4cqBMnTrjHTJgwQW+//baWLVumDz/8UCdPntR111131icDAAAAAAAAAPCfs7m8EXAm3vr7Ve9LaV177bW69tpra3zNNE3NmTNHjz32mIYOHSpJWrx4sZKSkvTaa6/pzjvvVEFBgV555RW9+uqrGjBggCRpyZIlatu2rf7f//t/GjRo0FmcDgAAAAAAAAAAwOl59Q44OTk5ys3NVXp6untfWFiYrrzySm3ZskWSlJ2drYqKCo8xrVu31sUXX+we83NlZWUqLCz0eAAAAABAbd7564t65XdP64O3lvq7FAAAADRRaWlpmjBhgr/L8LpXXnnF42fwvtahQwfNmTPntK/n5eUpISFBBw8ebLSaauPVYCQ3N1eSlJSU5LE/KSnJ/Vpubq5CQ0PVokWL0475uenTpysuLs79aNu2rTfLBgAAABCEDn94RKURPfXNmj3+LgUAAACot40bN+rGG29UcnKyoqKi1KNHD/39738/43FlZWWaMmWKJk+eLMkVWhiGcdpHWlqaj89ESkxM1KhRo/T444/7fK268GowUu3n1/kyTfOM1/6qbcykSZNUUFDgfhw4cMBrtQIAAAAITqYZ5tpw2PxbCAAAAFCLioqKGvdv2bJF3bt314oVK/T5559r9OjRuu222/TPf/6z1vlWrFih6Oho9e/fX5K0detWHT58WIcPH9aKFSskSV999ZV731tvveXdEzqN22+/XX//+9+Vn5/fKOvVxqvBiN1ul6RTOj/y8vLcXSR2u13l5eWnnPx/j/m5sLAwxcbGejwAAAAAoFbVwYgZ6t86AAAAEDCWLFmi1NRUxcTEyG63a+TIkcrLy5Pk+uX+Tp06adasWR7H7N69WxaLRd9++60kqaCgQOPGjVNiYqJiY2N11VVXaefOne7xWVlZ6tGjhxYsWKCOHTsqLCxMpmmeUsujjz6qJ598Uv369dN5552n+++/X9dcc43efvvtWs9h2bJluuGGG9zPExISZLfbZbfb1bJlS0muDg673a6EhAT9/ve/V0pKiiIiItSlSxfNnTvXY77MzEwNGTJEs2bNUnJysuLj43XPPfecEugUFxdr9OjRiomJUbt27fTSSy95vN6tWzfZ7fYz1t8YvBqMpKSkyG63a/369e595eXl2rRpk/r16ydJuvTSS2Wz2TzGHD58WLt373aPAQAAAICzZriCEdNJMAIAANDYTNNUcXmlXx41hQx1VV5erieffFI7d+7UypUrlZOTo8zMTEmuKyWNHj1aCxcu9DhmwYIF6t+/v8477zyZpqmMjAzl5uZqzZo1ys7OVq9evXT11Vfr2LFj7mO++eYbLV++XCtWrNCOHTvqXF9BQYE73DidzZs3KzU1tU7zOZ1OnXvuuVq+fLn27NmjKVOm6NFHH9Xy5cs9xm3YsEHffvutNmzYoMWLF2vRokVatGiRx5inn35aqamp2r59u8aPH6+7775bX375pceY3r17a/PmzXWqzZdC6nvAyZMn9c0337if5+TkaMeOHWrZsqXatWunCRMmaNq0aercubM6d+6sadOmKTIyUiNHjpQkxcXFacyYMfrf//1fxcfHq2XLlnrooYfUrVs3DRgwwHtnBgAAAKBZM1XVMeL+CgAAgMZSUuFQ1ylr/bL2nicGKTK03j/6liSNHj3avd2xY0c9++yz6t27t06ePKno6GjdfvvtmjJlij799FP17t1bFRUVWrJkiWbOnCnJFSDs2rVLeXl5Cgtz/T901qxZWrlypd58802NGzdOkiuAefXVV5WQkFDn2t58801t3bpVf/nLX0475vjx4zp+/Lhat25dpzltNpumTp3qfp6SkqItW7Zo+fLlGjZsmHt/ixYtNG/ePFmtVl1wwQXKyMjQe++9p7Fjx7rHDB48WOPHj5ckPfzww5o9e7Y2btyoCy64wD2mTZs22r59e53P2Vfq/bdj27Zt+tWvfuV+/uCDD0qSfve732nRokWaOHGiSkpKNH78eOXn56tPnz5at26dYmJi3MfMnj1bISEhGjZsmEpKSnT11Vdr0aJFslqtXjglAAAAAJBMS1UgYhCMAAAAoG62b9+urKws7dixQ8eOHZPT6ZQk7d+/X127dlVycrIyMjK0YMEC9e7dW6tXr1ZpaaluvvlmSVJ2drZOnjyp+Ph4j3lLSkrcl9qSpPbt29crFNm4caMyMzP18ssv66KLLjrtuJKSEklSeHh4ned+8cUX9de//lX79u1TSUmJysvL1aNHD48xF110kcfP75OTk7Vr1y6PMd27d3dvG4Yhu93uvgxZtYiICBUXF9e5Nl+pdzCSlpZWayuSYRjKyspSVlbWaceEh4frueee03PPPVff5QEAAACgTpyG68OgSTACAADQ6CJsVu15YpDf1m6IoqIipaenKz09XUuWLFFCQoL279+vQYMGqby83D3ujjvu0KhRozR79mwtXLhQw4cPV2RkpCTXpamSk5O1cePGU+Y/55xz3NtRUVF1rmvTpk26/vrr9cwzz+i2226rdWx8fLwMw6jzDc6XL1+uBx54QE8//bT69u2rmJgYzZw5U5988onHOJvN5vHcMAx3aFSfMceOHatXIOQrDesnAgAAAIAmzml1BSPVAQkAAAAaj2EYDb6clb98+eWXOnLkiJ566im1bdtWkusKSj83ePBgRUVFaf78+XrnnXf0wQcfuF/r1auXcnNzFRISog4dOpx1TRs3btR1112nP//5z+7LcNUmNDRUXbt21Z49e5Senn7G8Zs3b1a/fv3cl8CS5NHZ4m27d+9WWlqaz+avK6/efB0AAAAAmoLy0lI5ra5OEaeFjhEAAACcWbt27RQaGqrnnntO3333nVatWqUnn3zylHFWq1WZmZmaNGmSOnXqpL59+7pfGzBggPr27ashQ4Zo7dq12rt3r7Zs2aI//OEPNYYstdm4caMyMjJ0//336ze/+Y1yc3OVm5vrcRP3mgwaNEgffvhhndbo1KmTtm3bprVr1+rrr7/W5MmTtXXr1nrVWVfFxcXKzs6uU2DjawQjAAAAAIJO7v7v3NvVnSMAAABAbRISErRo0SK98cYb6tq1q5566inNmjWrxrFjxoxReXm5x83aJVenzJo1a3TFFVdo9OjROv/88zVixAjt3btXSUlJ9apn0aJFKi4u1vTp05WcnOx+DB06tNbjxo4dqzVr1qigoOCMa9x1110aOnSohg8frj59+ujo0aMe3SPe9I9//EPt2rVT//79fTJ/fRhmbTcMaaIKCwsVFxengoICxcbG+rscAAAAAE3MZ++v08fLf7p0w8gnu6pFgt2PFQEAAAS30tJS5eTkKCUlpV43/g5UH330kdLS0vT999/XO/BoDMOGDVPPnj01adIkf5fi1rt3b02YMEEjR45s8By1/T2rT25AxwgAAACAoHM87weP5z/s3+enSgAAABBMysrK9M0332jy5MkaNmxYkwxFJGnmzJmKjo72dxlueXl5uummm3TLLbf4uxRJBCMAAAAAglDRseMez/MPHfJPIQAAAAgqS5cuVZcuXVRQUKAZM2b4u5zTat++ve677z5/l+GWmJioiRMnyjAMf5ciiWAEAAAAQBAqLTzp8bzwyBE/VQIAAIBgkpmZKYfDoezsbLVp08bf5aCBCEYAAAAABJ2KolKP5yXHC/1UCQAAAICmhmAEAAAAQNCpLKn0eF52othPlQAAAABoaghGAAAAAAQdZ5np8byipNxPlQAAAABoaghGAAAAAAQds8Lzpo6OUoefKgEAAADQ1BCMAAAAAAg6ZqXV47mzzE+FAAAAAGhyCEYAAAAABB9HiMdTs5KPPgAAAABc+HQAAAAAIOiYTpvnDoe15oEAAABo1tLS0jRhwgR/l+F1r7zyitLT0xttvQ4dOmjOnDmnfT0vL08JCQk6ePBgo9VUG4IRAAAAAMHHDJUkWSuLXM8dtloGAwAAAE3P3r17ZRjGKY9333231uPKyso0ZcoUTZ48WZIrtKhpnupHWlqaz88lMTFRo0aN0uOPP+7zteoi5MxDAAAAACDQhEmSQioL5AiJcgclAAAAQFNTUVEhm+30v8jz//7f/9NFF13kft6yZcta51uxYoWio6PVv39/SdLWrVvlcDgkSVu2bNFvfvMbffXVV4qNjZUkhYY2zv+Vb7/9dvXu3VszZ85UixYtGmXN06FjBAAAAEDwMVzBiOEskCSZBCMAAACogyVLlig1NVUxMTGy2+0aOXKk8vLyJEmmaapTp06aNWuWxzG7d++WxWLRt99+K0kqKCjQuHHjlJiYqNjYWF111VXauXOne3xWVpZ69OihBQsWqGPHjgoLC5NpmqetKT4+Xna73f04U5CxbNky3XDDDe7nCQkJ7mOrQ5XExETZ7XYlJCTo97//vVJSUhQREaEuXbpo7ty5HvNlZmZqyJAhmjVrlpKTkxUfH6977rlHFRUVHuOKi4s1evRoxcTEqF27dnrppZc8Xu/WrZvsdrvefvvtWutvDAQjAAAAAIKOWRWMyDhRtSfMb7UAAAA0S6YplRf551FLyHAm5eXlevLJJ7Vz506tXLlSOTk5yszMlCQZhqHRo0dr4cKFHscsWLBA/fv313nnnSfTNJWRkaHc3FytWbNG2dnZ6tWrl66++modO3bMfcw333yj5cuXa8WKFdqxY0etNd1www1KTEzU5ZdfrjfffPOM57B582alpqbW6XydTqfOPfdcLV++XHv27NGUKVP06KOPavny5R7jNmzYoG+//VYbNmzQ4sWLtWjRIi1atMhjzNNPP63U1FRt375d48eP1913360vv/zSY0zv3r21efPmOtXmS1xKCwAAAEDQMY1wSZIRUlz1nGAEAACgUVUUS9Na+2ftRw9JoVENOnT06NHu7Y4dO+rZZ59V7969dfLkSUVHR+v222/XlClT9Omnn6p3796qqKjQkiVLNHPmTEmuAGHXrl3Ky8tTWJjr/6CzZs3SypUr9eabb2rcuHGSXAHMq6++qoSEhNPWEh0drWeeeUaXX365LBaLVq1apeHDh2vx4sW69dZbazzm+PHjOn78uFq3rtv33mazaerUqe7nKSkp2rJli5YvX65hw4a597do0ULz5s2T1WrVBRdcoIyMDL333nsaO3ase8zgwYM1fvx4SdLDDz+s2bNna+PGjbrgggvcY9q0aaPt27fXqTZfIhgBAAAAEHSclqpLaUVUSJU/BSUAAABAbbZv366srCzt2LFDx44dk9PplCTt379fXbt2VXJysjIyMrRgwQL17t1bq1evVmlpqW6++WZJUnZ2tk6ePKn4+HiPeUtKStyX2pKk9u3b1xqKSFKrVq30wAMPuJ+npqYqPz9fM2bMOG0wUlJSIkkKD6/7/39ffPFF/fWvf9W+fftUUlKi8vJy9ejRw2PMRRddJKvV6n6enJysXbt2eYzp3r27e9swDNntdvdlyKpFRESouLi4zrX5CsEIAAAAgKDjsLo+CIbG2VR8VDItdIwAAAA0Klukq3PDX2s3QFFRkdLT05Wenq4lS5YoISFB+/fv16BBg1ReXu4ed8cdd2jUqFGaPXu2Fi5cqOHDhysy0rWm0+lUcnKyNm7ceMr855xzjns7KqphHS2XXXaZ/vrXv5729fj4eBmGofz8/DrNt3z5cj3wwAN6+umn1bdvX8XExGjmzJn65JNPPMb9/ObwhmG4Q6P6jDl27NgZA6HGQDACAAAAIKicOH5cpsX1oSza3kLHj0pOCx0jAAAAjcowGnw5K3/58ssvdeTIET311FNq27atJGnbtm2njBs8eLCioqI0f/58vfPOO/rggw/cr/Xq1Uu5ubkKCQlRhw4dvF7j9u3blZycfNrXQ0ND1bVrV+3Zs0fp6elnnG/z5s3q16+f+xJYkjw6W7xt9+7dSktL89n8dcXN1wEAAAAElR/2fePeTup0niTJYQ1TZUWFv0oCAABAAGjXrp1CQ0P13HPP6bvvvtOqVav05JNPnjLOarUqMzNTkyZNUqdOndS3b1/3awMGDFDfvn01ZMgQrV27Vnv37tWWLVv0hz/8ocaQpTaLFy/Wa6+9pi+++EJfffWVZs2apWeffVb33XdfrccNGjRIH374YZ3W6NSpk7Zt26a1a9fq66+/1uTJk7V169Z61VlXxcXFys7OrlNg42sEIwAAAACCypFDrks2GM4KtbvgQtdOw6Ijh/b7sSoAAAA0dQkJCVq0aJHeeOMNde3aVU899ZRmzZpV49gxY8aovLzc42btkuvyUWvWrNEVV1yh0aNH6/zzz9eIESO0d+9eJSUl1bumP/7xj0pNTdUvfvELLVu2TAsWLPC470hNxo4dqzVr1qigoOCM8991110aOnSohg8frj59+ujo0aMe3SPe9I9//EPt2rVT//79fTJ/fRimaZr+LqK+CgsLFRcXp4KCAsXGxvq7HAAAAABNyPq/vaKvt6QopOKkxrx4rf5y7ybJsKj/KEPdL/+Vv8sDAAAISqWlpcrJyVFKSkq9bvwdqD766COlpaXp+++/b1Dg4WvDhg1Tz549NWnSJH+X4ta7d29NmDBBI0eObPActf09q09uQMcIAAAAgKBSlF8oSbI4SxVis8nqKJMkHf8h159lAQAAIAiUlZXpm2++0eTJkzVs2LAmGYpI0syZMxUdHe3vMtzy8vJ000036ZZbbvF3KZIIRgAAAAAEmfKTRZIkw3QFIhan6+uJI/l+qwkAAADBYenSperSpYsKCgo0Y8YMf5dzWu3btz/jvUgaU2JioiZOnCjDMPxdiiSCEQAAAABBpqLIFYQYZqnrq9P1tbTgpN9qAgAAQHDIzMyUw+FQdna22rRp4+9y0EAEIwAAAACCSmWJw7VhVgckrq/lJ0v8VRIAAACAJoRgBAAAAEBQcVZUbRhVwYhcXytLKk5zBAAAAIDmhGAEAAAAQFAxy6uvW1xetcMVjDjKnP4pCAAAAECTQjACAAAAIKiYjhBJkmGp6hCp6hwxy/1VEQAAAICmhGAEAAAAQHBxuoIRWSslSYbFlYiYlVZ/VQQAAACgCSEYAQAAABBcnKGuryFVN2E3XJ0j1Z0kAAAAAJo3ghEAAAAAQcYVjFhspuup1RWQGE6CEQAAAHhKS0vThAkT/F2G173yyitKT09vtPU6dOigOXPmnPb1vLw8JSQk6ODBg41WU20IRgAAAAAEFzNMkmQJc92E3bA5PfYDAAAAgWLt2rW67LLLFBMTo4SEBP3mN79RTk5OrceUlZVpypQpmjx5siRXaGEYxmkfaWlpPj+PxMREjRo1So8//rjP16oLghEAAAAAQcU0XAFISISrQ8RSdWUt0wz1V0kAAADAaVVUVNS4/7vvvtONN96oq666Sjt27NDatWt15MgRDR06tNb5VqxYoejoaPXv31+StHXrVh0+fFiHDx/WihUrJElfffWVe99bb73l3RM6jdtvv11///vflZ+f3yjr1YZgBAAAAEBwMcIlSbYo11drWPVN1+kYAQAAQO2WLFmi1NRUxcTEyG63a+TIkcrLy5MkmaapTp06adasWR7H7N69WxaLRd9++60kqaCgQOPGjVNiYqJiY2N11VVXaefOne7xWVlZ6tGjhxYsWKCOHTsqLCxMpmmeUstnn30mh8OhP/7xjzrvvPPUq1cvPfTQQ9q5c+dpwxRJWrZsmW644Qb384SEBNntdtntdrVs2VKSq4PDbrcrISFBv//975WSkqKIiAh16dJFc+fO9ZgvMzNTQ4YM0axZs5ScnKz4+Hjdc889p9RQXFys0aNHKyYmRu3atdNLL73k8Xq3bt1kt9v19ttvn7b2xkIwAgAAACCoOC2uACQ8NlqSFBJlk/RTJwkAAAB8zzRNFVcU++VRU8hQV+Xl5XryySe1c+dOrVy5Ujk5OcrMzJQkGYah0aNHa+HChR7HLFiwQP3799d5550n0zSVkZGh3NxcrVmzRtnZ2erVq5euvvpqHTt2zH3MN998o+XLl2vFihXasWNHjbWkpqbKarVq4cKFcjgcKigo0Kuvvqr09HTZbLbTnsPmzZuVmppap/N1Op0699xztXz5cu3Zs0dTpkzRo48+quXLl3uM27Bhg7799ltt2LBBixcv1qJFi7Ro0SKPMU8//bRSU1O1fft2jR8/Xnfffbe+/PJLjzG9e/fW5s2b61SbL3H3QQAAAABBxWlxdYpEtTxHkhQWEylJMqv2AwAAwPdKKkvU57U+fln7k5GfKNIW2aBjR48e7d7u2LGjnn32WfXu3VsnT55UdHS0br/9dk2ZMkWffvqpevfurYqKCi1ZskQzZ86U5AoQdu3apby8PIWFuX4xZ9asWVq5cqXefPNNjRs3TpIrgHn11VeVkJBw2lo6dOigdevW6eabb9add94ph8Ohvn37as2aNac95vjx4zp+/Lhat25dp/O12WyaOnWq+3lKSoq2bNmi5cuXa9iwYe79LVq00Lx582S1WnXBBRcoIyND7733nsaOHeseM3jwYI0fP16S9PDDD2v27NnauHGjLrjgAveYNm3aaPv27XWqzZfoGAEAAAAQNCorKuSwuj6AnpOYJEmKiIuVJDkNghEAAADUbvv27brxxhvVvn17xcTEuG9Mvn//fklScnKyMjIytGDBAknS6tWrVVpaqptvvlmSlJ2drZMnTyo+Pl7R0dHuR05OjvtSW5LUvn37WkMRScrNzdUdd9yh3/3ud9q6das2bdqk0NBQ3XTTTaftiikpKZEkhYfX/f++L774olJTU5WQkKDo6Gi9/PLL7vOtdtFFF8lqtbqfJycnuy8xVq179+7ubcMwZLfbTxkTERGh4uLiOtfmK3SMAAAAAAga+T/mSobrA1urc8+VJMXEx0uSnFYupQUAANBYIkIi9MnIT/y2dkMUFRUpPT1d6enpWrJkiRISErR//34NGjRI5eXl7nF33HGHRo0apdmzZ2vhwoUaPny4IiNdHSpOp1PJycnauHHjKfOfc8457u2oqKgz1vP8888rNjZWM2bMcO9bsmSJ2rZtq08++USXXXbZKcfEx8fLMIw63+B8+fLleuCBB/T000+rb9++iomJ0cyZM/XJJ55/dj+/dJdhGHI6nfUec+zYsTMGQo2BYAQAAABA0Phh3z73tr1dR0lSy2S7pDI5rWEqKSpSRB0+hAIAAODsGIbR4MtZ+cuXX36pI0eO6KmnnlLbtm0lSdu2bTtl3ODBgxUVFaX58+frnXfe0QcffOB+rVevXsrNzVVISIg6dOhwVvUUFxd7dGlIcj//eeBQLTQ0VF27dtWePXuUnp5+xjU2b96sfv36uS+BJcmjs8Xbdu/e7e7C8ScupQUAAAAgaBzPOyxJsjjKFFp1+YDEdh3cr/9wIMcfZQEAACAAtGvXTqGhoXruuef03XffadWqVXryySdPGWe1WpWZmalJkyapU6dO6tu3r/u1AQMGqG/fvhoyZIjWrl2rvXv3asuWLfrDH/5QY8hSm4yMDG3dulVPPPGE/vOf/+izzz7T7bffrvbt26tnz56nPW7QoEH68MMP67RGp06dtG3bNq1du1Zff/21Jk+erK1bt9arzroqLi5WdnZ2nQIbXyMYAQAAABA0Tvx4VJJkcZS698XFJ8pwVkqSjh484Je6AAAA0PQlJCRo0aJFeuONN9S1a1c99dRTmjVrVo1jx4wZo/Lyco+btUuuTpk1a9boiiuu0OjRo3X++edrxIgR2rt3r5KSkupVz1VXXaXXXntNK1euVM+ePXXNNdcoLCxM7777riIiTn+5sLFjx2rNmjUqKCg44xp33XWXhg4dquHDh6tPnz46evSoR/eIN/3jH/9Qu3bt1L9/f5/MXx+Gebq7tDRQZWWlsrKy9Pe//125ublKTk5WZmam/vCHP8hiceUwpmlq6tSpeumll5Sfn68+ffro+eef10UXXVSnNQoLCxUXF6eCggLFxsZ6s3wAAAAAAeytZ2bq8NeXKrTsB41deIt7/4tjVslhi9aFV36vq265zY8VAgAABKfS0lLl5OQoJSWlXjf+DlQfffSR0tLS9P3339c78GgMw4YNU8+ePTVp0iR/l+LWu3dvTZgwQSNHjmzwHLX9PatPbuD1jpE///nPevHFFzVv3jx98cUXmjFjhmbOnKnnnnvOPWbGjBl65plnNG/ePG3dulV2u10DBw7UiRMnvF0OAAAAgGak/ESJJMlwlnnst1Q9LzpWt5tQAgAAADUpKyvTN998o8mTJ2vYsGFNMhSRpJkzZyo6OtrfZbjl5eXppptu0i233HLmwY3A68HIxx9/rBtvvFEZGRnq0KGDbrrpJqWnp7uvn2aapubMmaPHHntMQ4cO1cUXX6zFixeruLhYr732mrfLAQAAANCMVBaXS5IM/SwYMV2X1iotLGr0mgAAABA8li5dqi5duqigoEAzZszwdzmn1b59e913333+LsMtMTFREydOlGEY/i5Fkg+CkV/+8pd677339PXXX0uSdu7cqQ8//FCDBw+WJOXk5Cg3N9fjBithYWG68sortWXLlhrnLCsrU2FhoccDAAAAAH7OUep0bZiewYhR9byiqOznhwAAAAB1lpmZKYfDoezsbLVp08bf5aCBQrw94cMPP6yCggJdcMEFslqtcjgc+tOf/uRukcnNzZWkU1qMkpKStG/fvhrnnD59uqZOnertUgEAAAAEGWdF1YbxswCkKhipLK1s3IIAAAAANDle7xh5/fXXtWTJEr322mv67LPPtHjxYs2aNUuLFy/2GPfzlhnTNE/bRjNp0iQVFBS4HwcOHPB22QAAAACCgFlR9RHHKP/ZK65gxFlmNm5BAAAAAJocr3eM/P73v9cjjzyiESNGSJK6deumffv2afr06frd734nu90uydU5kpyc7D4uLy/vtDeqCQsLU1hYmLdLBQAAABBsHCGSVTIsFR67DYsrKHFWNI1rGgMAAADwH693jBQXF8ti8ZzWarXK6XRd6zclJUV2u13r1693v15eXq5NmzapX79+3i4HAAAAQDNiOm2uDavD84XqDpJKa+MWBAAAAKDJ8XrHyPXXX68//elPateunS666CJt375dzzzzjEaPHi3JdQmtCRMmaNq0aercubM6d+6sadOmKTIyUiNHjvR2OQAAAACaEzPU9dX2s2DEWnVvEYfXPwIBAAAACDBe/1Tw3HPPafLkyRo/frzy8vLUunVr3XnnnZoyZYp7zMSJE1VSUqLx48crPz9fffr00bp16xQTE+PtcgAAAAA0J1XBiNX2s/0hrqDE3VECAAAAoNny+qW0YmJiNGfOHO3bt08lJSX69ttv9cc//lGhoaHuMYZhKCsrS4cPH1Zpaak2bdqkiy++2NulAAAAAGh2XPcmtIR7ftSxhFTfdD1UAAAAQLW0tDRNmDDB32X4xeTJkzVu3LhGW88wDK1cufK0r+/atUvnnnuuioqKfF6L14MRAAAAAPAX0wiXJNkiPDtDLGFVN103wxq7JAAAAKBBSktLlZmZqW7duikkJERDhgw5Zczhw4c1cuRIdenSRRaLpc4hzw8//KC5c+fq0UcfleQKLWp7ZGZmeu/ETqNbt27q3bu3Zs+e7fO1CEYAAAAABA3TcAUftugIj/3W8KqrCBsEIwAAAGhaKioqatzvcDgUERGh+++/XwMGDKhxTFlZmRISEvTYY4/pkksuqfOar7zyivr27asOHTpIcgUs1Y85c+YoNjbWY9/cuXPrfV4Ncfvtt2v+/PlyOBxnHnwWCEYAAAAABA3T4go+IuI8718YGuXaX91RAgAAANRkyZIlSk1NVUxMjOx2u0aOHKm8vDxJkmma6tSpk2bNmuVxzO7du2WxWPTtt99KkgoKCjRu3DglJiYqNjZWV111lXbu3Oken5WVpR49emjBggXq2LGjwsLCZJqmfi4qKkrz58/X2LFjZbfba6y3Q4cOmjt3rm677TbFxcXV+TyXLVumG264wf3cbre7H3FxcTIMw/3cZrPprrvu0rnnnqvIyEh169ZNS5cu9ZgvLS1N999/vyZOnKiWLVvKbrcrKyvrlHWPHDmiX//614qMjFTnzp21atUqj9cHDRqko0ePatOmTXU+l4YgGAEAAAAQNBxWV/AREx/vsT8sJlLSTx0lAAAA8C3TNOUsLvbLo6aQoa7Ky8v15JNPaufOnVq5cqVycnLcl5EyDEOjR4/WwoULPY5ZsGCB+vfvr/POO0+maSojI0O5ublas2aNsrOz1atXL1199dU6duyY+5hvvvlGy5cv14oVK7Rjx44G19sQ+fn52r17t1JTU+s0vrS0VJdeeqlWr16t3bt3a9y4cRo1apQ++eQTj3GLFy9WVFSUPvnkE82YMUNPPPGE1q9f7zFm6tSpGjZsmD7//HMNHjxYv/3tbz2+L6Ghobrkkku0efPmsz/RWoT4dHYAAAAAaCTlpaVyVgUjcfYkj9eiWp4jSXJa6BgBAABoDGZJib7qdalf1u7yWbaMyMgGHTt69Gj3dseOHfXss8+qd+/eOnnypKKjo3X77bdrypQp+vTTT9W7d29VVFRoyZIlmjlzpiRpw4YN2rVrl/Ly8hQW5vqlnFmzZmnlypV688033Tc7Ly8v16uvvqqEhISzPNv627dvn0zTVOvWres0vk2bNnrooYfcz++77z69++67euONN9SnTx/3/u7du+vxxx+XJHXu3Fnz5s3Te++9p4EDB7rHZGZm6pZbbpEkTZs2Tc8995w+/fRTXXPNNR7r7d2792xO8YzoGAEAAAAQFH44sM+9bW/bweO12MREST91lAAAAAA12b59u2688Ua1b99eMTExSktLkyTt379fkpScnKyMjAwtWLBAkrR69WqVlpbq5ptvliRlZ2fr5MmTio+PV3R0tPuRk5PjvtSWJLVv394voYgklZSUSJLCw+v2f2OHw6E//elP6t69u/u81q1b5/6eVOvevbvH8+TkZPdlyGoaExUVpZiYmFPGREREqLi4uM7n0xB0jAAAAAAICkcPHpBkkUynWto9f/stvvW5ko7ItIQo/8dctUio+RrNAAAA8A4jIkJdPsv229oNUVRUpPT0dKWnp2vJkiVKSEjQ/v37NWjQIJWXl7vH3XHHHRo1apRmz56thQsXavjw4Yqs6lBxOp1KTk7Wxo0bT5n/nHPOcW9HRUU1qEZvaNWqlSTXJbXqEs48/fTTmj17tubMmaNu3bopKipKEyZM8PieSJLNZvN4bhiGnE5nvcccO3ZM5513Xp3PpyEIRgAAAAAEheN5P0hKltVRqpCffeBKatte0hFJ0g/79xGMAAAA+JhhGA2+nJW/fPnllzpy5IieeuoptW3bVpK0bdu2U8YNHjzYfWP0d955Rx988IH7tV69eik3N1chISHq0KFDY5VeL+edd55iY2O1Z88enX/++Wccv3nzZt1444269dZbJbnCn//85z+68MILfVLf7t27ddNNN/lk7mpcSgsAAABAUDh5LF+SZHGWnfJaVGycLA7Xb7TlHzrUqHUBAAAgMLRr106hoaF67rnn9N1332nVqlV68sknTxlntVqVmZmpSZMmqVOnTurbt6/7tQEDBqhv374aMmSI1q5dq71792rLli36wx/+UGPIciZ79uzRjh07dOzYMRUUFGjHjh2n3Ky9et/Jkyf1448/aseOHdqzZ89p57RYLBowYIA+/PDDOtXQqVMnrV+/Xlu2bNEXX3yhO++8U7m5ufU+l7rYu3evDh48qAEDBvhk/moEIwAAAACCQmnBCUmS4Syt8XVL1f7CI0carSYAAAAEjoSEBC1atEhvvPGGunbtqqeeekqzZs2qceyYMWNUXl7ucbN2ydUps2bNGl1xxRUaPXq0zj//fI0YMUJ79+5VUlJSvWsaPHiwevbsqX/+85/auHGjevbsqZ49e3qMqd6XnZ2t1157TT179tTgwYNrnXfcuHFatmzZKZexqsnkyZPVq1cvDRo0SGlpabLb7RoyZEi9z6Uuli5dqvT0dLVv394n81czTNM0fbqCDxQWFiouLk4FBQWKjY31dzkAAAAAmoClU7J0LO8KhZXs1R2LR5/y+su3v67ysAS16bJdQx74Xz9UCAAAELxKS0uVk5OjlJSUOt/UO5B99NFHSktL0/fff9+gwMPfTNPUZZddpgkTJuiWW27xdzmSpLKyMnXu3FlLly7V5ZdfXuOY2v6e1Sc3oGMEAAAAQFCoLK6QJBnmqZfSkn7qJCk7UdxoNQEAACC4lJWV6ZtvvtHkyZM1bNiwgAxFJFdny0svvaTKykp/l+K2b98+PfbYY6cNRbyJm68DAAAACAqOsurLAJwmGKnaX1FS3kgVAQAAINgsXbpUY8aMUY8ePfTqq6/6u5yzcskll+iSSy7xdxlu559/fp1uBu8NdIwAAAAACApmpeHaMGoORlTVSeIodTRSRQAAAAg2mZmZcjgcys7OVps2bfxdDhqIYAQAAABAUDArrJIkw6ioeUBVYOI8TW4CAAAAoHkgGAEAAAAQHJxVVwq21hyMGIbrElpmJR+DAAAAgOaMTwQAAAAAgoLpDHVtWGu+gaRpqQpMHNZGqggAAABAU0QwAgAAACA4mK5gxLCZNb5sVAcmDltjVQQAAACgCSIYAQAAABAcqoIRS2jNLxshDo9xAAAAAJonghEAAAAAwcEIkySFhNd8qazqThKTYAQAAABo1ghGAAAAAAQFU+GSpJCosBpft4RVf/yp+XUAAAA0P2lpaZowYYK/y/CLyZMna9y4cY22nmEYWrly5Wlf37Vrl84991wVFRX5vBaCEQAAAABBwbS4Ao+w6MgaX7dFujpFTINgBAAAAE1faWmpMjMz1a1bN4WEhGjIkCG1jv/oo48UEhKiHj16nHHuH374QXPnztWjjz4qyRVa1PbIzMw8+xM6g27duql3796aPXu2z9ciGAEAAAAQFJwWV8dIZIvYGl+3VXWSmEZ4o9UEAAAAnElFRUWN+x0OhyIiInT//fdrwIABtc5RUFCg2267TVdffXWd1nzllVfUt29fdejQQZJ0+PBh92POnDmKjY312Dd37tx6nVND3X777Zo/f74cDodP1yEYAQAAABAUnFUdIzGtWtX4ekRcjKSfOksAAACAn1uyZIlSU1MVExMju92ukSNHKi8vT5JkmqY6deqkWbNmeRyze/duWSwWffvtt5JcIcW4ceOUmJio2NhYXXXVVdq5c6d7fFZWlnr06KEFCxaoY8eOCgsLk2map9QSFRWl+fPna+zYsbLb7bXWfeedd2rkyJHq27dvnc5z2bJluuGGG9zP7Xa7+xEXFyfDMNzPbTab7rrrLp177rmKjIxUt27dtHTpUo/50tLSdP/992vixIlq2bKl7Ha7srKyTln3yJEj+vWvf63IyEh17txZq1at8nh90KBBOnr0qDZt2lSn82goghEAAAAAAa+osEBOq+tSWfHJrWscEx3fQtJPnSUAAADwHdM0VVHm8MujppChrsrLy/Xkk09q586dWrlypXJyctyXkTIMQ6NHj9bChQs9jlmwYIH69++v8847T6ZpKiMjQ7m5uVqzZo2ys7PVq1cvXX311Tp27Jj7mG+++UbLly/XihUrtGPHjgbXK0kLFy7Ut99+q8cff7xO4/Pz87V7926lpqbWaXxpaakuvfRSrV69Wrt379a4ceM0atQoffLJJx7jFi9erKioKH3yySeaMWOGnnjiCa1fv95jzNSpUzVs2DB9/vnnGjx4sH772996fF9CQ0N1ySWXaPPmzXWqraFCfDo7AAAAADSCw/u+c28nte9Y45hzkly/ZeewhqmyokIhNluj1AYAANAcVZY79dL/+Pa3/k9n3NwrZQuzNujY0aNHu7c7duyoZ599Vr1799bJkycVHR2t22+/XVOmTNGnn36q3r17q6KiQkuWLNHMmTMlSRs2bNCuXbuUl5ensDBXp/KsWbO0cuVKvfnmm+6bnZeXl+vVV19VQkLCWZ3rf/7zHz3yyCPavHmzQkLq9uP+ffv2yTRNtW5d8y8U/VybNm300EMPuZ/fd999evfdd/XGG2+oT58+7v3du3d3hzOdO3fWvHnz9N5772ngwIHuMZmZmbrlllskSdOmTdNzzz2nTz/9VNdcc43Henv37q1TbQ1FxwgAAACAgHfs8EFJkuGsUFx8zZfSSji3rWvDsOjIof2NVRoAAAACyPbt23XjjTeqffv2iomJUVpamiRp/37X/x+Tk5OVkZGhBQsWSJJWr16t0tJS3XzzzZKk7OxsnTx5UvHx8YqOjnY/cnJy3JfakqT27dufdSjicDg0cuRITZ06Veeff36djyspKZEkhYfXrZPa4XDoT3/6k7p37+4+r3Xr1rm/J9W6d+/u8Tw5Odl9GbKaxkRFRSkmJuaUMRERESouLq7z+TQEHSMAAAAAAl5h3hFJ7WR1lJ12TOK5HSRzn2RYlPf9ftnbn9do9QEAADQ3IaEWjZt7pd/WboiioiKlp6crPT1dS5YsUUJCgvbv369BgwapvLzcPe6OO+7QqFGjNHv2bC1cuFDDhw9XZGSkJMnpdCo5OVkbN248Zf5zzjnHvR0VFdWgGv/biRMntG3bNm3fvl333nuve33TNBUSEqJ169bpqquuOuW4VlX35MvPz69TOPP0009r9uzZmjNnjrp166aoqChNmDDB43siSbafdWQbhiGn01nvMceOHdN55/n2/+oEIwAAAAACXlF+vqR2sjhLTzsmxGaT1VEmR0iEjv+Q23jFAQAANEOGYTT4clb+8uWXX+rIkSN66qmn1Latq9t427Ztp4wbPHiw+8bo77zzjj744AP3a7169VJubq5CQkLUoUMHn9YbGxurXbt2eex74YUX9P777+vNN99USkpKjcedd955io2N1Z49e+rUabJ582bdeOONuvXWWyW5wpf//Oc/uvDCC8/+JGqwe/du3XTTTT6ZuxqX0gIAAAAQ8MoKiyRJhnn6jhFJsjhdr584ku/zmgAAABBY2rVrp9DQUD333HP67rvvtGrVKj355JOnjLNarcrMzNSkSZPUqVMn9e3b1/3agAED1LdvXw0ZMkRr167V3r17tWXLFv3hD3+oMWQ5kz179mjHjh06duyYCgoKtGPHDvfN2i0Wiy6++GKPR2JiosLDw3XxxReftivFYrFowIAB+vDDD+tUQ6dOnbR+/Xpt2bJFX3zxhe68807l5vrmF4327t2rgwcPasCAAT6ZvxrBCAAAAICAV1HkCjwM8/QdI5JkVHWUlBac9HlNAAAACCwJCQlatGiR3njjDXXt2lVPPfWUZs2aVePYMWPGqLy83ONm7ZKrU2bNmjW64oorNHr0aJ1//vkaMWKE9u7dq6SkpHrXNHjwYPXs2VP//Oc/tXHjRvXs2VM9e/Zs0Pn9t3HjxmnZsmWnXMaqJpMnT1avXr00aNAgpaWlyW63a8iQIWddQ02WLl2q9PR0tW/f3ifzVzNM0zR9uoIPFBYWKi4uTgUFBYqNjfV3OQAAAAD8bPEDj+pkyQCFF+/RmL/de9pxf/3dQpVFtFfLpM26ZerjjVghAABAcCstLVVOTo5SUlLqfFPvQPbRRx8pLS1N33//fYMCD38zTVOXXXaZJkyYoFtuucXf5UiSysrK1LlzZy1dulSXX355jWNq+3tWn9yAjhEAAAAAAc9ZUbVh1H4pLUOu1ytLKmodBwAAANSkrKxM33zzjSZPnqxhw4YFZCgiuTpbXnrpJVVWVvq7FLd9+/bpscceO20o4k3cfB0AAABAwDPLjapf+yo/w0BXMOIoO/MlAwAAAICfW7p0qcaMGaMePXro1Vdf9Xc5Z+WSSy7RJZdc4u8y3M4///w63QzeG+gYAQAAABDwTIfrd74Myxk6Qao6Sswz5CcAAABATTIzM+VwOJSdna02bdr4uxw0EMEIAAAAgMDnrGqGt9Z+KQDD4kpEzEqrrysCAAAA0EQRjAAAAAAIfM5Q19cQR+3jDFdHSXWHCQAAALzLNE1/l4Ag5q2/XwQjAAAAAIKAKxix2M7wQcnqCk4MJ8EIAACAN1mtro7c8nKuWQrfqf77Vf33raH4NAAAAAAg8JlhkiRLmFHrMMPmlJw/jQcAAIB3hISEKDIyUj/++KNsNpssFn4nH97ldDr1448/KjIyUiEhZxdtEIwAAAAACHim4Qo6QsJr/4hjCZVUKplmaCNUBQAA0HwYhqHk5GTl5ORo3759/i4HQcpisahdu3YyjNp/IepMCEYAAAAABD4jXJJki6q9E8QaZpVKJYmOEQAAAG8LDQ1V586duZwWfCY0NNQr3UgEIwAAAAACntPiCjrCY6NqHRcSZZMKfuowAQAAgHdZLBaFh4f7uwygVlzoDQAAAEDAc1pcH76jWraodVxYTKQkybTwYR0AAABorghGAAAAAAS0yooKOayuDpC4xIRax0bExUqSnAbBCAAAANBcEYwAAAAACGj5P+ZKhlWSFN+mba1jY+LjJUlOK5fSAgAAAJornwQjBw8e1K233qr4+HhFRkaqR48eys7Odr9umqaysrLUunVrRUREKC0tTf/+9799UQoAAACAIPfDvn3u7aS2KbWObZlsl+QKRkqKinxaFwAAAICmyevBSH5+vi6//HLZbDa988472rNnj55++mmdc8457jEzZszQM888o3nz5mnr1q2y2+0aOHCgTpw44e1yAAAAAAS543mHJUkWR5kiomq/+Xpiuw7u7R8O5PiyLAAAAABNVIi3J/zzn/+stm3bauHChe59HTp0cG+bpqk5c+boscce09ChQyVJixcvVlJSkl577TXdeeed3i4JAAAAQBA78eNRSfGyOErPODYuPlGGc49Mi1VHDx5Qhwsu9n2BAAAAAJoUr3eMrFq1Sqmpqbr55puVmJionj176uWXX3a/npOTo9zcXKWnp7v3hYWF6corr9SWLVtqnLOsrEyFhYUeDwAAAACQpJJCV+e5xTxzMBJis8nidI0ryPvRp3UBAAAAaJq8Hox89913mj9/vjp37qy1a9fqrrvu0v3336+//e1vkqTc3FxJUlJSksdxSUlJ7td+bvr06YqLi3M/2rat/YaKAAAAAJqP8hMlkiTDWVan8RaHa1zRsXyf1QQAAACg6fJ6MOJ0OtWrVy9NmzZNPXv21J133qmxY8dq/vz5HuMMw/B4bprmKfuqTZo0SQUFBe7HgQMHvF02AAAAgABVWVwuSTJUx2CkqrOktJCbrwMAAADNkdeDkeTkZHXt2tVj34UXXqj9+/dLkux2uySd0h2Sl5d3ShdJtbCwMMXGxno8AAAAAECSHKVO14ZZt2CkelxFUR3HAwAAAAgqXg9GLr/8cn311Vce+77++mu1b99ekpSSkiK73a7169e7Xy8vL9emTZvUr18/b5cDAAAAIMg5K6o2jLoFHUZVx0hlaaWPKgIAAADQlIV4e8IHHnhA/fr107Rp0zRs2DB9+umneumll/TSSy9Jcl1Ca8KECZo2bZo6d+6szp07a9q0aYqMjNTIkSO9XQ4AAACAIGdWWCSrJKO8jke4xjnLTJ/VBAAAAKDp8now8otf/EJvv/22Jk2apCeeeEIpKSmaM2eOfvvb37rHTJw4USUlJRo/frzy8/PVp08frVu3TjExMd4uBwAAAECwc4RIVsmwVJx5rOTuLHFW1HyPQwAAAADBzevBiCRdd911uu666077umEYysrKUlZWli+WBwAAANCMmE6ba8PqqNsB1QFKpdU3BQEAAABo0rx+jxEAAAAAaFRmqOurrW7BiLuzxOGT3xMDAAAA0MQRjAAAAAAIbFXBiNVWt+FGiCtAcXeaAAAAAGhWCEYAAAAABLgwSZIlvI4fb2zOqo1Q35QDAAAAoEkjGAEAAAAQ0EwjXJJki6hbB4g1tOqm62aYr0oCAAAA0IQRjAAAAAAIaKbhCjhs0RF1Gm+NqLq3iEEwAgAAADRHBCMAAAAAApppcQUcEXExdRpvi3SNr+40AQAAANC8EIwAAAAACGgOqyvgiImPr9P4sNhIST91mgAAAABoXghGAAAAAASs8tJSOauCkTh7Up2OiWrRQpLktNAxAgAAADRHBCMAAAAAAtYPB/a5t+1tO9TpmNjEVpIkh5WOEQAAAKA5IhgBAAAAELCOHjzg2jCdamlvXadjWia3cR1isang6BFflQYAAACgiSIYAQAAABCwjuf9IEmyOkoVYrPV6Zjk9h3d2z/s+84ndQEAAABoughGAAAAAASsk8fyJUkWZ1mdj4mKjZPFUS5JOnr4kE/qAgAAANB0EYwAAAAACFilBSckSYaztF7HVQcpJ45wKS0AAACguSEYAQAAABCwKopcAYdh1r1jRJIsVUFKcX6h12sCAAAA0LQRjAAAAAAIWJXFFZLqH4wYVR0jZSeLvV4TAAAAgKaNYAQAAABAwHKUOau26hmMyNUxUllUv+MAAAAABD6CEQAAAAABy6w0XBtGPQOOqg6TylKHlysCAAAA0NQRjAAAAAAIWGaFVZJkGBX1O7AqSHGWe7siAAAAAE0dwQgAAACAwOUMcX211i8YMQxXImJWGN6uCAAAAEATRzACAAAAIGCZzlDXhrWyfsdZqoIUR4iXKwIAAADQ1BGMAAAAAAhcpisYMWxmvQ4zqoMUh83bFQEAAABo4ghGAAAAAASuqmDEElq/w4wQh8fxAAAAAJoPghEAAAAAgcsIkySFhFvrd1hVh4lJMAIAAAA0OwQjAAAAAAKWqXBJUkhUWL2Os4RVfxSq33EAAAAAAh/BCAAAAICAZVpcwUZYdGS9jrNFujpFTINgBAAAAGhuCEYAAAAABCynxdUxEtkitl7H2ao6TEwj3Os1AQAAAGjaCEYAAAAABCxnVcdITKtW9TouIi5G0k8dJwAAAACaD4IRAAAAAAGpqLBATqvrkljxya3rdWx0fAtJP3WcAAAAAGg+CEYAAAAABKTD+75zbye171ivY89JskuSHNYwVVZUeLUuAAAAAE0bwQgAAACAgHTs8EFJkuGsUFx8/S6llXBuW9eGYdGRQ/u9XRoAAACAJoxgBAAAAEBAKsw7IkmyOsrqfWziuR0k0ylJyvueYAQAAABoTghGAAAAAASkovx8SZLFWVrvY0NsNnegcvyHXK/WBQAAAKBpIxgBAAAAEJDKCoslSYZZ/44RSbI4XcedOJLvtZoAAAAANH0EIwAAAAACUkWxK9gwzPp3jEiSUdVpUlpw0ms1AQAAAGj6CEYAAAAABCRHSaVro4EdI9WdJuUnS7xVEgAAAIAAQDACAAAAICA5yk3XhtHAYESu4ypLKrxVEgAAAIAAQDACAAAAIDBVVH+cKW/Y8VUdI44yp3fqAQAAABAQCEYAAAAABCSz0ipJMiwN7Pio6jQxG5irAAAAAAhMBCMAAAAAApLptLk2rJUNOt6wuBKR6oAFAAAAQPNAMAIAAAAgMFUHIyGOhh1vuDpNTEeIlwoCAAAAEAgIRgAAAAAEJjNMkmSxmQ073uoKVAwnwQgAAADQnBCMAAAAAAhQoZIkS5jRoKMNW9VN16sCFgAAAADNA8EIAAAAgIBkGuGSpJDwhnV8WEKr5jFDvVUSAAAAgABAMAIAAAAgMBmuTg9bVMM6Pqxh1Tddp2MEAAAAaE4IRgAAAAAEJGdVx0h4bFSDjg+Jct283TQIRgAAAIDmxOfByPTp02UYhiZMmODeZ5qmsrKy1Lp1a0VERCgtLU3//ve/fV0KAAAAgCDitLoCjaiWLRp0fFhMpCTJtIR7rSYAAAAATZ9Pg5GtW7fqpZdeUvfu3T32z5gxQ88884zmzZunrVu3ym63a+DAgTpx4oQvywEAAAAQJCorKuSsCjTiEhMaNEdEXKyknzpPAAAAADQPPgtGTp48qd/+9rd6+eWX1aLFT7/BZZqm5syZo8cee0xDhw7VxRdfrMWLF6u4uFivvfaar8oBAAAAEEQKjubJtLjuERLfpm2D5oiJj5f0U+cJAAAAgObBZ8HIPffco4yMDA0YMMBjf05OjnJzc5Wenu7eFxYWpiuvvFJbtmypca6ysjIVFhZ6PAAAAAA0X3n797q3k9qmNGiOlsl2Sa5gpKSoyBtlAQAAAAgAPglGli1bps8++0zTp08/5bXc3FxJUlJSksf+pKQk92s/N336dMXFxbkfbds27DfCAAAAAASHY4ddnx0sjjJFRDXs5uuJ7Tq4t384kOONsgAAAAAEAK8HIwcOHND//M//aMmSJQoPP/21eg3D8HhumuYp+6pNmjRJBQUF7seBAwe8WjMAAACAwHLi6FFJrmCkoeLiE2U4HZKkowf5jAEAAAA0FyHenjA7O1t5eXm69NJL3fscDoc++OADzZs3T1999ZUkV+dIcnKye0xeXt4pXSTVwsLCFBbGdX8BAAAAuJQUuC6vazFLGzxHiM0mi7NUDkuUCvJ+9FZpAAAAAJo4r3eMXH311dq1a5d27NjhfqSmpuq3v/2tduzYoY4dO8put2v9+vXuY8rLy7Vp0yb169fP2+UAAAAACEJlJ4olSYaz4cGI9FPHSdGx/LOuCQAAAEBg8HrHSExMjC6++GKPfVFRUYqPj3fvnzBhgqZNm6bOnTurc+fOmjZtmiIjIzVy5EhvlwMAAAAgCFUWVUiSDDX8UlrSTx0npYXcfB0AAABoLrwejNTFxIkTVVJSovHjxys/P199+vTRunXrFBMT449yAAAAAAQYR5nr3iAyzy4YqT6+ougs5wEAAAAQMBolGNm4caPHc8MwlJWVpaysrMZYHgAAAECQcZZXbRhnF2gYVR0jlaWVZ1kRAAAAgEDh9XuMAAAAAICvmRVVH2WM8toHnpHreGeZeZbzAAAAAAgUBCMAAAAAAo/DKkkyLBVnN09Vx4mzwjjbigAAAAAECIIRAAAAAIHHtLm+Wh1nN091sFJpPbt5AAAAAAQMghEAAAAAAcd0hro2bGcXjLg7ThyNcvtFAAAAAE0AwQgAAACAwGOGSZKstrObxghxBSum8ywnAgAAABAwCEYAAAAABB7DFYxYws/yI43NWbURenbzAAAAAAgYBCMAAAAAAo4pVzBiizi7Tg9raNVN16s6UAAAAAAEP4IRAAAAAAHHrOoYsUVHnNU81oiqe4sYBCMAAABAc0EwAgAAACDgmJZwSVJEXMxZzWOLdAUiphF+1jUBAAAACAwEIwAAAAACjtPiCjRi4uPPap6w2EhJP3WgAAAAAAh+BCMAAAAAAkplRYUcVleQEWdPOqu5olq0kCQ5LXSMAAAAAM0FwQgAAACAgJL3/V7JcH2UsbftcFZzxSa2kiR30AIAAAAg+BGMAAAAAAgoP35/wLVhOtXS3vqs5mqZ3MY1lcWmgqNHzrY0AAAAAAGAYAQAAABAQMk/fFiSZHWUKsRmO6u5ktt3dG//sO+7s5oLAAAAQGAgGAEAAAAQUE4ey5ckWZxlZz1XVGycLI5ySdLRw4fOej4AAAAATR/BCAAAAICAUlpwQpJkOEu9Ml91wHLiCJfSAgAAAJoDghEAAAAAAaWiyBVkGObZd4xIkqUqYCnOL/TKfAAAAACaNoIRAAAAAAGlsrhCkveCEaOqY6TsZLFX5gMAAADQtBGMAAAAAAgojjJn1ZaXghG5OkYqi7wzHwAAAICmjWAEAAAAQEAxKw3XhuGlIKOq86Sy1OGd+QAAAAA0aQQjAAAAAAKKWWGVJBlGhXcmNMolSc5y70wHAAAAoGkjGAEAAAAQWJwhrq9W7wYjZoXhnfkAAAAANGkEIwAAAAACiukMdW1YK70yn2GpahVxhHhlPgAAAABNG8EIAAAAgMBiuoIRw2Z6Zz5LVcDisHlnPgAAAABNGsEIAAAAgMBSFYxYQr0znWFz3XTdNAlGAAAAgOaAYAQAAABAYDHCJEkh4VbvTBdS1XlihnllPgAAAABNG8EIAAAAgIBiKlySFBLlnSDDGlb9sYhgBAAAAGgOCEYAAAAABBTT4gowwqIjvTJfSKTrElqmQTACAAAANAcEIwAAAAACitPi6hiJbBHrlflsVZ0nBCMAAABA80AwAgAAACCgOKs6RmJatfLKfOFxMZIksypwAQAAABDcCEYAAAAABIyiwgI5raGSpPjk1l6ZM7plC0k/BS4AAAAAghvBCAAAAICAcXjfd+7tpPYdvTLnOYlJkiSHNVyVFRVemRMAAABA00UwAgAAACBgHDt8UJJkOCsUF++dS2nFt2nr2jAsOpZ7yCtzAgAAAGi6CEYAAAAABIzCvCOSJKujzGtzJrVt797OPbDXa/MCAAAAaJoIRgAAAAAEjKL8fEmSxVnqtTlDw8NlcbjmK8j9wWvzAgAAAGiaCEYAAAAABIyywmJJkmF6r2NEkqxVwciJo0e9Oi8AAACApodgBAAAAEDAqCh2BSKG6b2OEUkynK55SwpOeHVeAAAAAE0PwQgAAACAgOEoqXRteLljpLoDpfxEiVfnBQAAAND0EIwAAAAACBiOctO1YXg7GHF1oFSWVHh1XgAAAABND8EIAAAAgMBRUf0RptzLE7uCFkeZ08vzAgAAAGhqCEYAAAAABAyz0ipJMixe7uwwXEGL6e28BQAAAECTQzACAAAAIGCYTptrw1rp3Ymrg5Gq4AUAAABA8CIYAQAAABA4qoOREIdXp63uQDEdIV6dFwAAAEDTQzACAAAAIHCYYZIki8307rxVHSiGk2AEAAAACHYEIwAAAAACSKgkyRJmeHVWw1Z10/Wq4AUAAABA8PJ6MDJ9+nT94he/UExMjBITEzVkyBB99dVXHmNM01RWVpZat26tiIgIpaWl6d///re3SwEAAAAQZEwjXJIUEu7dzg5L1RW6TDPUq/MCAAAAaHq8Hoxs2rRJ99xzj/71r39p/fr1qqysVHp6uoqKitxjZsyYoWeeeUbz5s3T1q1bZbfbNXDgQJ04ccLb5QAAAAAIJoaro8MW5d3ODmt49UcjOkYAAACAYOf1C+i+++67Hs8XLlyoxMREZWdn64orrpBpmpozZ44ee+wxDR06VJK0ePFiJSUl6bXXXtOdd97p7ZIAAAAABAlnVcdIeGyUV+cNiQqVCiTTIBgBAAAAgp3P7zFSUFAgSWrZsqUkKScnR7m5uUpPT3ePCQsL05VXXqktW7bUOEdZWZkKCws9HgAAAACaH6fVFVxEtWzh1XnDYiIlSaYl3KvzAgAAAGh6fBqMmKapBx98UL/85S918cUXS5Jyc3MlSUlJSR5jk5KS3K/93PTp0xUXF+d+tG3b1pdlAwAAAGiCKisq5KwKLuISE7w6d0RcrKSfOlIAAAAABC+fBiP33nuvPv/8cy1duvSU1wzD8HhumuYp+6pNmjRJBQUF7seBAwd8Ui8AAACApqvgaJ5Mi1WSFN/Gu78sFRMfL+mnjhQAAAAAwcvr9xipdt9992nVqlX64IMPdO6557r32+12Sa7OkeTkZPf+vLy8U7pIqoWFhSksjA8oAAAAQHOWt3+vezupbYpX526ZbJdUJqc1TCVFRYqI8u49TAAAAAA0HV7vGDFNU/fee6/eeustvf/++0pJ8fzAkpKSIrvdrvXr17v3lZeXa9OmTerXr5+3ywEAAAAQJI4ddl161+Io83pwkdiug3v7hwM5Xp0bAAAAQNPi9WDknnvu0ZIlS/Taa68pJiZGubm5ys3NVUlJiSTXJbQmTJigadOm6e2339bu3buVmZmpyMhIjRw50tvlAAAAAAgSJ44eleQKRrwtLj5RhtMhSTp6kEv3AgAAAMHM65fSmj9/viQpLS3NY//ChQuVmZkpSZo4caJKSko0fvx45efnq0+fPlq3bp1iYmK8XQ4AAACAIFFSUChJspilXp87xGaTxVkqhyVKBXk/en1+AAAAAE2H14MR0zTPOMYwDGVlZSkrK8vbywMAAAAIUmUniiVJhtP7wYjk6kRxhESp6Fi+T+YHAAAA0DR4/VJaAAAAAOALlUUVkiTD9P6ltKSfOlFKC4t8Mj8AAACApoFgBAAAAEBAcJQ5qrZ8E4yoKnCpKPLR/AAAAACaBIIRAAAAAAHBWe76ahjlPpnfqOoYqSyt9Mn8AAAAAJoGghEAAAAAAcGsqPr4Yviqo8MVuDjLznzfRAAAAACBi2AEAAAAQGBwWCVJpsVHHR1VgYuzwvDN/AAAAACaBIIRAAAAAIHBtEmSDKuPghGL6+buqrT6Zn4AAAAATQLBCAAAAICAYDpDXRs2R+0DG8ioDkYcIT6ZHwAAAEDTQDACAAAAIDCYYZIkq8030xshrsDFdPpoAQAAAABNAsEIAAAAgMBguIIRS7iPPsbYnFUbob6ZHwAAAECTQDACAAAAICCYcgUjtgjfdHRYQ6tuul7VmQIAAAAgOBGMAAAAAAgIZlXHiC06wifzWyOq7i1iEIwAAAAAwYxgBAAAAEBAMC3hkqSIuBifzG+LdAUiphHuk/kBAAAANA0EIwAAAAACgtPiCi5i4uN9Mn9YbKSknzpTAAAAAAQnghEAAAAATV5lRYUcVldgEWdP8skaUS1aSJKcFjpGAAAAgGBGMAIAAACgycv7fq9kuD6+2Nt28MkasYmtJMkdwAAAAAAITgQjAAAAAJq8H78/4NownWppb+2TNVomt3EtYbGp4OgRn6wBAAAAwP8IRgAAAAA0ecd/yJUkWR2lCrHZfLJGcvuO7u0f9n3nkzUAAAAA+B/BCAAAAIAm7+TRfEmSxVnmszWiYuNkcZRLko4ePuSzdQAAAAD4F8EIAAAAgCavpOCEJMlwlvp0nerg5cQRLqUFAAAABCuCEQAAAABNXkWRK7AwTN91jEiSpSp4Kc4v9Ok6AAAAAPyHYAQAAABAk1dR7LrEla+DEaOqY6TsZLFP1wEAAADgPwQjAAAAAJo8Z5mzasvHwYhcHSOVRb5dBwAAAID/EIwAAAAAaPLMCsO1Yfg4sKjqSKksdfh2HQAAAAB+QzACAAAAoMkzK62SJMOo8O1ChuuSXc5y3y4DAAAAwH8IRgAAAAA0eaYjxLVhbZxgxN2hAgAAACDoEIwAAAAAaPIMp821Ya307TqWqlaR6iAGAAAAQNAhGAEAAADQ5JlmqCTJsJm+XchSFbw4bL5dBwAAAIDfEIwAAAAAaPrMMEmSJdS3yxg2103XTZNgBAAAAAhWBCMAAAAAmj7DFYyEhFt9u0xIVUdKVRADAAAAIPgQjAAAAABo8kxVBSNRvg0srGHVH5EIRgAAAIBgRTACAAAAoMkzLeGSpLDoSJ+uExLpuoSWaRCMAAAAAMGKYAQAAABAk+e0uIKKyBaxPl3HVtWRQjACAAAABC+CEQAAAABNnrOqYySmVSufrhMeFyPppw4VAAAAAMGHYAQAAABAk1ZUWCCnNVSSFJ/c2qdrRbdsIemnDhUAAAAAwYdgBAAAAECT9sOBfe7tpPYdfbrWOYlJkiSHNVyVFRU+XQsAAACAfxCMAAAAAGjSjh76XpJkOCsUF+/bS2nFt2nr2jAsOpZ7yKdrAQAAAPAPghEAAAAATVphXp4kyeoo8/laSW3bu7dzD+z1+XoAAAAAGh/BCAAAAIAmrejYcUmSxVnq87VCw8NlcbjWKcj9wefrAQAAAGh8BCMAAAAAmrSyE8WSJMP0fceIJFmrgpETR482ynoAAAAAGhfBCAAAAIAmrbzIFYgYpu87RiTJcLrWKyk40SjrAQAAAGhcBCMAAAAAmjRHaaVro5E6Rqo7UypOljTKegAAAAAaF8EIAAAAgCbNWWa6NozGCkZcnSkVJRWNsh4AAACAxkUwAgAAAKBJc1YaVVvljbSiK4BxljobaT0AAAAAjYlgBAAAAEDTVmmVJBmWRurgMFwBjIOGEQAAACAoEYwAAAAAaNocIa6v1srGWa8qGFGFtXHWAwAAANCo/BqMvPDCC0pJSVF4eLguvfRSbd682Z/lAAAAAGiKzFDX1xBHoyzn7kxxEIwAAAAAwchvwcjrr7+uCRMm6LHHHtP27dvVv39/XXvttdq/f7+/SgIAAADQBJlOVzBisZmNs2BVZ4rptDXOegAAAAAald+CkWeeeUZjxozRHXfcoQsvvFBz5sxR27ZtNX/+fH+VBAAAAKBJCpMkWcKMM4zzDsNWddP16k4VAAAAAEElxB+LlpeXKzs7W4888ojH/vT0dG3ZsuWU8WVlZSorK3M/Lyws9HmNgejdBS/p4KbG+bAIAAAANJaK0A6SpJDwxvn4YrFJKpMqQrrold+93ChrAgAAAPVilmnM3+71dxUByy/ByJEjR+RwOJSUlOSxPykpSbm5uaeMnz59uqZOndpY5QWs4uMFKo241N9lAAAAAD6RcH67Rlkn+txzdPJbyWGLlsMW3ShrAgAAAPVhrSz2dwkBzS/BSDXD8OxuME3zlH2SNGnSJD344IPu54WFhWrbtq3P6ws0Xa+8QiVH/8/fZQAAAABeF9c2UQNva5zfiLtxwgNa/cI8FeXlN8p6AAAAQH1ZQqySrvN3GQHLL8FIq1atZLVaT+kOycvLO6WLRJLCwsIUFhbWWOUFrAsu7aMLLu3j7zIAAACAgBZis2nI/zzg7zIAAAAA+Ihfbr4eGhqqSy+9VOvXr/fYv379evXr188fJQEAAAAAAAAAgGbAb5fSevDBBzVq1Cilpqaqb9++eumll7R//37ddddd/ioJAAAAAAAAAAAEOb8FI8OHD9fRo0f1xBNP6PDhw7r44ou1Zs0atW/f3l8lAQAAAAAAAACAIGeYpmn6u4j6KiwsVFxcnAoKChQbG+vvcgAAAAAAAAAAgB/VJzfwyz1GAAAAAAAAAAAA/IFgBAAAAAAAAAAANBsEIwAAAAAAAAAAoNkgGAEAAAAAAAAAAM0GwQgAAAAAAAAAAGg2CEYAAAAAAAAAAECzEeLvAhrCNE1JUmFhoZ8rAQAAAAAAAAAA/ladF1TnB7UJyGDkxIkTkqS2bdv6uRIAAAAAAAAAANBUnDhxQnFxcbWOMcy6xCdNjNPp1KFDhxQTEyPDMPxdDhDwCgsL1bZtWx04cECxsbH+LgdAkOM9B0Bj4f0GQGPiPQdAY+I9BziVaZo6ceKEWrduLYul9ruIBGTHiMVi0bnnnuvvMoCgExsbyz+mABoN7zkAGgvvNwAaE+85ABoT7zmApzN1ilTj5usAAAAAAAAAAKDZIBgBAAAAAAAAAADNBsEIAIWFhenxxx9XWFiYv0sB0AzwngOgsfB+A6Ax8Z4DoDHxngOcnYC8+ToAAAAAAAAAAEBD0DECAAAAAAAAAACaDYIRAAAAAAAAAADQbBCMAAAAAAAAAACAZoNgBAAAAAAAAAAANBsEI0AzlZ+fr1GjRikuLk5xcXEaNWqUjh8/Xufj77zzThmGoTlz5visRgDBob7vNxUVFXr44YfVrVs3RUVFqXXr1rrtttt06NChxisaQMB44YUXlJKSovDwcF166aXavHlzreM3bdqkSy+9VOHh4erYsaNefPHFRqoUQDCoz3vOW2+9pYEDByohIUGxsbHq27ev1q5d24jVAgh09f1/TrWPPvpIISEh6tGjh28LBAIYwQjQTI0cOVI7duzQu+++q3fffVc7duzQqFGj6nTsypUr9cknn6h169Y+rhJAMKjv+01xcbE+++wzTZ48WZ999pneeustff3117rhhhsasWoAgeD111/XhAkT9Nhjj2n79u3q37+/rr32Wu3fv7/G8Tk5ORo8eLD69++v7du369FHH9X999+vFStWNHLlAAJRfd9zPvjgAw0cOFBr1qxRdna2fvWrX+n666/X9u3bG7lyAIGovu851QoKCnTbbbfp6quvbqRKgcBkmKZp+rsIAI3riy++UNeuXfWvf/1Lffr0kST961//Ut++ffXll1+qS5cupz324MGD6tOnj9auXauMjAxNmDBBEyZMaKTKAQSas3m/+W9bt25V7969tW/fPrVr186XJQMIIH369FGvXr00f/58974LL7xQQ4YM0fTp008Z//DDD2vVqlX64osv3Pvuuusu7dy5Ux9//HGj1AwgcNX3PacmF110kYYPH64pU6b4qkwAQaKh7zkjRoxQ586dZbVatXLlSu3YsaMRqgUCDx0jQDP08ccfKy4uzv1DSkm67LLLFBcXpy1btpz2OKfTqVGjRun3v/+9LrroosYoFUCAa+j7zc8VFBTIMAydc845PqgSQCAqLy9Xdna20tPTPfanp6ef9v3l448/PmX8oEGDtG3bNlVUVPisVgCBryHvOT/ndDp14sQJtWzZ0hclAggiDX3PWbhwob799ls9/vjjvi4RCHgh/i4AQOPLzc1VYmLiKfsTExOVm5t72uP+/Oc/KyQkRPfff78vywMQRBr6fvPfSktL9cgjj2jkyJGKjY31dokAAtSRI0fkcDiUlJTksT8pKem07y+5ubk1jq+srNSRI0eUnJzss3oBBLaGvOf83NNPP62ioiINGzbMFyUCCCINec/5z3/+o0ceeUSbN29WSAg/8gXOhI4RIIhkZWXJMIxaH9u2bZMkGYZxyvGmada4X5Kys7M1d+5cLVq06LRjADQfvny/+W8VFRUaMWKEnE6nXnjhBa+fB4DA9/P3kjO9v9Q0vqb9AFCT+r7nVFu6dKmysrL0+uuv1/hLIwBQk7q+5zgcDo0cOVJTp07V+eef31jlAQGN+BAIIvfee69GjBhR65gOHTro888/1w8//HDKaz/++OMpv41QbfPmzcrLy/O4tr/D4dD//u//as6cOdq7d+9Z1Q4gsPjy/aZaRUWFhg0bppycHL3//vt0iwDw0KpVK1mt1lN+azIvL++07y92u73G8SEhIYqPj/dZrQACX0Pec6q9/vrrGjNmjN544w0NGDDAl2UCCBL1fc85ceKEtm3bpu3bt+vee++V5Lp8n2maCgkJ0bp163TVVVc1Su1AoCAYAYJIq1at1KpVqzOO69u3rwoKCvTpp5+qd+/ekqRPPvlEBQUF6tevX43HjBo16pT/xA8aNEijRo3S7bfffvbFAwgovny/kX4KRf7zn/9ow4YN/MASwClCQ0N16aWXav369fr1r3/t3r9+/XrdeOONNR7Tt29f/fOf//TYt27dOqWmpspms/m0XgCBrSHvOZKrU2T06NFaunSpMjIyGqNUAEGgvu85sbGx2rVrl8e+F154Qe+//77efPNNpaSk+LxmINAQjADN0IUXXqhrrrlGY8eO1V/+8hdJ0rhx43TdddepS5cu7nEXXHCBpk+frl//+teKj48/5QeTNptNdrvd4xgA+G8Neb+prKzUTTfdpM8++0yrV6+Ww+Fw/6ZUy5YtFRoa6pdzAdD0PPjggxo1apRSU1PVt29fvfTSS9q/f7/uuusuSdKkSZN08OBB/e1vf5Mk3XXXXZo3b54efPBBjR07Vh9//LFeeeUVLV261J+nASBA1Pc9Z+nSpbrttts0d+5cXXbZZe7/z0RERCguLs5v5wEgMNTnPcdisejiiy/2OD4xMVHh4eGn7AfgQjACNFN///vfdf/99ys9PV2SdMMNN2jevHkeY7766isVFBT4ozwAQaS+7zfff/+9Vq1aJUnq0aOHx7gNGzYoLS3N5zUDCAzDhw/X0aNH9cQTT+jw4cO6+OKLtWbNGrVv316SdPjwYe3fv989PiUlRWvWrNEDDzyg559/Xq1bt9azzz6r3/zmN/46BQABpL7vOX/5y19UWVmpe+65R/fcc497/+9+9zstWrSoscsHEGDq+54DoH4Ms/pugwAAAAAAAAAAAEHO4u8CAAAAAAAAAAAAGgvBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJoNghEAAAAAAAAAANBsEIwAAAAAAAAAAIBmg2AEAAAAAAAAAAA0GwQjAAAAAAAAAACg2SAYAQAAAAAAAAAAzQbBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJqN/w/iTU6o6atUFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 2000x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize_histograms(layers, C, Xdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e32ba333",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of layers are :  17\n",
      "Total number of parameters are : 47551\n",
      "Epoch 0 Loss 3.295837163925171\n",
      "Epoch 100 Loss 3.063408613204956\n",
      "Epoch 200 Loss 2.780902862548828\n",
      "Epoch 300 Loss 3.1825616359710693\n",
      "Epoch 400 Loss 2.9627223014831543\n",
      "Epoch 500 Loss 2.983510971069336\n",
      "Epoch 600 Loss 2.810227394104004\n",
      "Epoch 700 Loss 3.0428051948547363\n",
      "Epoch 800 Loss 2.9010205268859863\n",
      "Epoch 900 Loss 2.6343629360198975\n"
     ]
    }
   ],
   "source": [
    "C, layers, losses, uds, v = train(Xtr, Ytr, 1000, print_at_every_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34fec83f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 2 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 5 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 8 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 11 (      Tanh): mean +0.000000, std 0.000000e+00\n",
      "layer 14 (      Tanh): mean +0.000000, std 0.000000e+00\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABkYAAAF0CAYAAAB2XoXVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABsHElEQVR4nO3deXhU5d3/8c+ZyWRfhJBkArIEQRQFAVMQLJoqBCUulCogFRtBUHF50MeiaMGgLVhAAUWxWlkqFURRSnlQ4KeAKFYhAkJxqRoWgRiBkED2zJzfH5OMHQkhCTOZzOT9uq65cubMfe77ewLXwOST7zmGaZqmAAAAAAAAAAAAmgGLvwsAAAAAAAAAAABoLAQjAAAAAAAAAACg2SAYAQAAAAAAAAAAzQbBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJoNghEAAAAAAAAAANBsEIwAAAAAAAAAAIBmg2AEAAAAAAAAAAA0GwQjAAAAQDPVoUMHZWZmup9v3LhRhmFo48aNPl33hRde0KJFi856HsMwlJWV5X6+aNEiGYahvXv31mueadOmaeXKlfU6pqa10tLSdPHFF9drnjNZs2aNxzn+t5//+QEAAACoG4IRAAAAAJKkXr166eOPP1avXr18uo63gpGfy8jI0Mcff6zk5OR6HdeQYKSha9XXmjVrNHXq1Bpfe/vttzV58mSfrg8AAAAEoxB/FwAAAACg7kzTVGlpqSIiIrw+d2xsrC677DKvz9tYEhISlJCQ4NM1SkpKFB4e3ihrnUnPnj39uj4AAAAQqOgYAQAAAPzgH//4h7p3766wsDB17NhRc+fOVVZWlgzD8BhnGIbuvfdevfjii7rwwgsVFhamxYsXS5KmTp2qPn36qGXLloqNjVWvXr30yiuvyDRNjzkqKio0ceJE2e12RUZG6pe//KU+/fTTU2o63aW0tm3bphtuuEEtW7ZUeHi4evbsqeXLl3uMqb601IYNG3T33XerVatWio+P19ChQ3Xo0CH3uA4dOujf//63Nm3aJMMwZBiGOnToUOv3qrCwUGPHjlV8fLyio6N1zTXX6Ouvvz5lXE2Xt9q+fbuuu+46JSYmKiwsTK1bt1ZGRoa+//579/e3qKhIixcvdteTlpbmMd+6des0evRoJSQkKDIyUmVlZbVetmvz5s267LLLFBERoTZt2mjy5MlyOBxn/D7v3btXhmG4u2kyMzP1/PPPu+usflSvWdOltPbv369bb73Vfb4XXnihnn76aTmdzlPWmTVrlp555hmlpKQoOjpaffv21b/+9a9a/ywAAACAYEDHCAAAANDI3n33XQ0dOlRXXHGFXn/9dVVWVmrWrFn64Ycfahy/cuVKbd68WVOmTJHdbldiYqIk1w+477zzTrVr106S9K9//Uv33XefDh48qClTpriPHzt2rP72t7/poYce0sCBA7V7924NHTpUJ06cOGOtGzZs0DXXXKM+ffroxRdfVFxcnJYtW6bhw4eruLj4lB/M33HHHcrIyNBrr72mAwcO6Pe//71uvfVWvf/++5Jcl3+66aabFBcXpxdeeEGSFBYWdtr1TdPUkCFDtGXLFk2ZMkW/+MUv9NFHH+naa689Y+1FRUUaOHCgUlJS9PzzzyspKUm5ubnasGGD+9w//vhjXXXVVfrVr37lvixVbGysxzyjR49WRkaGXn31VRUVFclms512zdzcXI0YMUKPPPKInnjiCf3f//2f/vjHPyo/P1/z5s07Y83/bfLkySoqKtKbb76pjz/+2L3/dJfv+vHHH9WvXz+Vl5frySefVIcOHbR69Wo99NBD+vbbb93f72rPP/+8LrjgAs2ZM8e93uDBg5WTk6O4uLh61QoAAAAEEoIRAAAAoJFNmTJFbdq00dq1axUaGipJuuaaa07bOXHy5Ent2rVLLVq08Ni/cOFC97bT6VRaWppM09TcuXM1efJkGYahL7/8UosXL9YDDzygGTNmSJIGDhyopKQk/fa3vz1jrePHj9dFF12k999/XyEhro8PgwYN0pEjR/Too4/qtttuk8XyUyP6Nddco2effdb9/NixY5o4caJyc3Nlt9vVs2dPRURE1PmyXWvXrtWGDRs0d+5c3X///e76Q0ND9dhjj9V67JdffqmjR4/qlVde0Y033ujeP2zYMPf2ZZddJovFooSEhNPWc/XVV+svf/nLGWuVpKNHj+of//iHbrjhBklSenq6SkpKNH/+fE2cONEdYtXFeeedp6SkJHedZ/LMM8/o4MGD+uSTT9S7d29Jrj8rh8OhF198URMmTND555/vHh8TE6PVq1fLarVKklq3bq3evXvrnXfe0YgRI+pcJwAAABBouJQWAAAA0IiKioq0bds2DRkyxB2KSFJ0dLSuv/76Go+56qqrTglFJOn999/XgAEDFBcXJ6vVKpvNpilTpujo0aPKy8uT5Or4kHRKCDJs2DB30HE633zzjb788kv3sZWVle7H4MGDdfjwYX311Vcex1QHAtW6d+8uSdq3b1+ta53O6eofOXLkGY/t1KmTWrRooYcfflgvvvii9uzZ06AafvOb39R5bExMzCnfg5EjR8rpdOqDDz5o0Pp19f7776tr167uUKRaZmamTNN0d+1Uy8jIcIci0tn/WQEAAACBgmAEAAAAaET5+fkyTdPdCfDfaton1XzppE8//VTp6emSpJdfflkfffSRtm7d6u6iKCkpkeTqYJAku93ucXxISIji4+NrrbX60l4PPfSQbDabx2P8+PGSpCNHjngc8/M5qy+TVV1PfR09erTGWn9+PjWJi4vTpk2b1KNHDz366KO66KKL1Lp1az3++OOqqKiocw2nu3RVTWr6M6yutfrPwleOHj1aY62tW7eucX1v/1kBAAAAgYJLaQEAAACNqEWLFjIMo8b7ieTm5tZ4zM9vyC5Jy5Ytk81m0+rVqxUeHu7ev3LlSo9x1T/8zs3NVZs2bdz7Kysrz/iD+latWkmSJk2apKFDh9Y4pkuXLrXOcbbi4+Pdtf73D/JP9736uW7dumnZsmUyTVOff/65Fi1apCeeeEIRERF65JFH6jRHTd//06ntz7W6/uo/r7KyMo9xPw+Z6is+Pl6HDx8+Zf+hQ4ck/fTnCQAAADR3dIwAAAAAjSgqKkqpqalauXKlysvL3ftPnjyp1atX13kewzAUEhLicSmkkpISvfrqqx7j0tLSJEl///vfPfYvX75clZWVta7RpUsXde7cWTt37lRqamqNj5iYmDrXXC0sLKzOXQm/+tWvaqz/tddeq9eahmHokksu0ezZs3XOOefos88+a1A9Z3LixAmtWrXqlFotFouuuOIKSXLfS+bzzz/3GPfz46prk+rWxXH11Vdrz549HucmSX/7299kGIb7ewkAAAA0d3SMAAAAAI3siSeeUEZGhgYNGqT/+Z//kcPh0MyZMxUdHa1jx47VaY6MjAw988wzGjlypMaNG6ejR49q1qxZ7h+kV7vwwgt16623as6cObLZbBowYIB2796tWbNmKTY29ozr/OUvf9G1116rQYMGKTMzU23atNGxY8f0xRdf6LPPPtMbb7xR7/Ov7uJ4/fXX1bFjR4WHh6tbt241jk1PT9cVV1yhiRMnqqioSKmpqfroo49OCYBqsnr1ar3wwgsaMmSIOnbsKNM09dZbb+n48eMaOHCgRz0bN27UP//5TyUnJysmJqbBnTDx8fG6++67tX//fp1//vlas2aNXn75Zd19993uG6/b7XYNGDBA06dPV4sWLdS+fXu99957euutt2r8XknSn//8Z1177bWyWq3q3r27x/1pqj3wwAP629/+poyMDD3xxBNq3769/u///k8vvPCC7r77bo8brwMAAADNGcEIAAAA0MiuueYarVixQlOmTNHw4cNlt9s1fvx4HTp0qE4/8JdcN2RfsGCB/vznP+v6669XmzZtNHbsWCUmJmrMmDEeY1955RUlJSVp0aJFevbZZ9WjRw+tWLFCI0aMOOM6v/rVr/Tpp5/qT3/6kyZMmKD8/HzFx8era9euGjZsWIPOf+rUqTp8+LDGjh2rEydOqH379tq7d2+NYy0Wi1atWqUHH3xQM2bMUHl5uS6//HKtWbNGF1xwQa3rdO7cWeecc45mzJihQ4cOKTQ0VF26dNGiRYv0u9/9zj1u7ty5uueeezRixAgVFxfryiuv1MaNGxt0bna7Xc8//7weeugh7dq1Sy1bttSjjz6qqVOneox79dVXdd999+nhhx+Ww+HQ9ddfr6VLlyo1NdVj3MiRI/XRRx/phRde0BNPPCHTNJWTk+PuOvlvCQkJ2rJliyZNmqRJkyapsLBQHTt21IwZM/Tggw826HwAAACAYGSYpmn6uwgAAACguauoqFCPHj3Upk0brVu3zt/lAAAAAEDQomMEAAAA8IMxY8Zo4MCBSk5OVm5url588UV98cUXmjt3rr9LAwAAAICgRjACAAAA+MGJEyf00EMP6ccff5TNZlOvXr20Zs0aDRgwwN+lAQAAAEBQ41JaAAAAAAAAAACg2bD4uwAAAAAAAAAAAIDGQjACAAAAAAAAAACaDYIRAAAAAAAAAADQbATkzdedTqcOHTqkmJgYGYbh73IAAAAAAAAAAIAfmaapEydOqHXr1rJYau8JCchg5NChQ2rbtq2/ywAAAAAAAAAAAE3IgQMHdO6559Y6JiCDkZiYGEmuE4yNjfVzNQAAAAAAAAAAwJ8KCwvVtm1bd35Qm4AMRqovnxUbG0swAgAAAAAAAAAAJKlOt9/g5usAAAAAAAAAAKDZIBgBAAAAAAAAAADNBsEIAAAAAAAAAABoNgLyHiMAAAAAAAAAgKbH6XSqvLzc32UgSIWGhspiOft+D4IRAAAAAAAAAMBZKy8vV05OjpxOp79LQZCyWCxKSUlRaGjoWc1DMAIAAAAAAAAAOCumaerw4cOyWq1q27atV36rH/hvTqdThw4d0uHDh9WuXTsZhtHguQhGAAAAAAAAAABnpbKyUsXFxWrdurUiIyP9XQ6CVEJCgg4dOqTKykrZbLYGz1Pv2O6DDz7Q9ddfr9atW8swDK1cudLjddM0lZWVpdatWysiIkJpaWn697//7TGmrKxM9913n1q1aqWoqCjdcMMN+v777xt8EgAAAAAAAAAA/3E4HJJ01pc4AmpT/fer+u9bQ9U7GCkqKtIll1yiefPm1fj6jBkz9Mwzz2jevHnaunWr7Ha7Bg4cqBMnTrjHTJgwQW+//baWLVumDz/8UCdPntR111131icDAAAAAAAAAPCfs7m8EXAm3vr7Ve9LaV177bW69tpra3zNNE3NmTNHjz32mIYOHSpJWrx4sZKSkvTaa6/pzjvvVEFBgV555RW9+uqrGjBggCRpyZIlatu2rf7f//t/GjRo0FmcDgAAAAAAAAAAwOl59Q44OTk5ys3NVXp6untfWFiYrrzySm3ZskWSlJ2drYqKCo8xrVu31sUXX+we83NlZWUqLCz0eAAAAABAbd7564t65XdP64O3lvq7FAAAADRRaWlpmjBhgr/L8LpXXnnF42fwvtahQwfNmTPntK/n5eUpISFBBw8ebLSaauPVYCQ3N1eSlJSU5LE/KSnJ/Vpubq5CQ0PVokWL0475uenTpysuLs79aNu2rTfLBgAAABCEDn94RKURPfXNmj3+LgUAAACot40bN+rGG29UcnKyoqKi1KNHD/39738/43FlZWWaMmWKJk+eLMkVWhiGcdpHWlqaj89ESkxM1KhRo/T444/7fK268GowUu3n1/kyTfOM1/6qbcykSZNUUFDgfhw4cMBrtQIAAAAITqYZ5tpw2PxbCAAAAFCLioqKGvdv2bJF3bt314oVK/T5559r9OjRuu222/TPf/6z1vlWrFih6Oho9e/fX5K0detWHT58WIcPH9aKFSskSV999ZV731tvveXdEzqN22+/XX//+9+Vn5/fKOvVxqvBiN1ul6RTOj/y8vLcXSR2u13l5eWnnPx/j/m5sLAwxcbGejwAAAAAoFbVwYgZ6t86AAAAEDCWLFmi1NRUxcTEyG63a+TIkcrLy5Pk+uX+Tp06adasWR7H7N69WxaLRd9++60kqaCgQOPGjVNiYqJiY2N11VVXaefOne7xWVlZ6tGjhxYsWKCOHTsqLCxMpmmeUsujjz6qJ598Uv369dN5552n+++/X9dcc43efvvtWs9h2bJluuGGG9zPExISZLfbZbfb1bJlS0muDg673a6EhAT9/ve/V0pKiiIiItSlSxfNnTvXY77MzEwNGTJEs2bNUnJysuLj43XPPfecEugUFxdr9OjRiomJUbt27fTSSy95vN6tWzfZ7fYz1t8YvBqMpKSkyG63a/369e595eXl2rRpk/r16ydJuvTSS2Wz2TzGHD58WLt373aPAQAAAICzZriCEdNJMAIAANDYTNNUcXmlXx41hQx1VV5erieffFI7d+7UypUrlZOTo8zMTEmuKyWNHj1aCxcu9DhmwYIF6t+/v8477zyZpqmMjAzl5uZqzZo1ys7OVq9evXT11Vfr2LFj7mO++eYbLV++XCtWrNCOHTvqXF9BQYE73DidzZs3KzU1tU7zOZ1OnXvuuVq+fLn27NmjKVOm6NFHH9Xy5cs9xm3YsEHffvutNmzYoMWLF2vRokVatGiRx5inn35aqamp2r59u8aPH6+7775bX375pceY3r17a/PmzXWqzZdC6nvAyZMn9c0337if5+TkaMeOHWrZsqXatWunCRMmaNq0aercubM6d+6sadOmKTIyUiNHjpQkxcXFacyYMfrf//1fxcfHq2XLlnrooYfUrVs3DRgwwHtnBgAAAKBZM1XVMeL+CgAAgMZSUuFQ1ylr/bL2nicGKTK03j/6liSNHj3avd2xY0c9++yz6t27t06ePKno6GjdfvvtmjJlij799FP17t1bFRUVWrJkiWbOnCnJFSDs2rVLeXl5Cgtz/T901qxZWrlypd58802NGzdOkiuAefXVV5WQkFDn2t58801t3bpVf/nLX0475vjx4zp+/Lhat25dpzltNpumTp3qfp6SkqItW7Zo+fLlGjZsmHt/ixYtNG/ePFmtVl1wwQXKyMjQe++9p7Fjx7rHDB48WOPHj5ckPfzww5o9e7Y2btyoCy64wD2mTZs22r59e53P2Vfq/bdj27Zt+tWvfuV+/uCDD0qSfve732nRokWaOHGiSkpKNH78eOXn56tPnz5at26dYmJi3MfMnj1bISEhGjZsmEpKSnT11Vdr0aJFslqtXjglAAAAAJBMS1UgYhCMAAAAoG62b9+urKws7dixQ8eOHZPT6ZQk7d+/X127dlVycrIyMjK0YMEC9e7dW6tXr1ZpaaluvvlmSVJ2drZOnjyp+Ph4j3lLSkrcl9qSpPbt29crFNm4caMyMzP18ssv66KLLjrtuJKSEklSeHh4ned+8cUX9de//lX79u1TSUmJysvL1aNHD48xF110kcfP75OTk7Vr1y6PMd27d3dvG4Yhu93uvgxZtYiICBUXF9e5Nl+pdzCSlpZWayuSYRjKyspSVlbWaceEh4frueee03PPPVff5QEAAACgTpyG68OgSTACAADQ6CJsVu15YpDf1m6IoqIipaenKz09XUuWLFFCQoL279+vQYMGqby83D3ujjvu0KhRozR79mwtXLhQw4cPV2RkpCTXpamSk5O1cePGU+Y/55xz3NtRUVF1rmvTpk26/vrr9cwzz+i2226rdWx8fLwMw6jzDc6XL1+uBx54QE8//bT69u2rmJgYzZw5U5988onHOJvN5vHcMAx3aFSfMceOHatXIOQrDesnAgAAAIAmzml1BSPVAQkAAAAaj2EYDb6clb98+eWXOnLkiJ566im1bdtWkusKSj83ePBgRUVFaf78+XrnnXf0wQcfuF/r1auXcnNzFRISog4dOpx1TRs3btR1112nP//5z+7LcNUmNDRUXbt21Z49e5Senn7G8Zs3b1a/fv3cl8CS5NHZ4m27d+9WWlqaz+avK6/efB0AAAAAmoLy0lI5ra5OEaeFjhEAAACcWbt27RQaGqrnnntO3333nVatWqUnn3zylHFWq1WZmZmaNGmSOnXqpL59+7pfGzBggPr27ashQ4Zo7dq12rt3r7Zs2aI//OEPNYYstdm4caMyMjJ0//336ze/+Y1yc3OVm5vrcRP3mgwaNEgffvhhndbo1KmTtm3bprVr1+rrr7/W5MmTtXXr1nrVWVfFxcXKzs6uU2DjawQjAAAAAIJO7v7v3NvVnSMAAABAbRISErRo0SK98cYb6tq1q5566inNmjWrxrFjxoxReXm5x83aJVenzJo1a3TFFVdo9OjROv/88zVixAjt3btXSUlJ9apn0aJFKi4u1vTp05WcnOx+DB06tNbjxo4dqzVr1qigoOCMa9x1110aOnSohg8frj59+ujo0aMe3SPe9I9//EPt2rVT//79fTJ/fRhmbTcMaaIKCwsVFxengoICxcbG+rscAAAAAE3MZ++v08fLf7p0w8gnu6pFgt2PFQEAAAS30tJS5eTkKCUlpV43/g5UH330kdLS0vT999/XO/BoDMOGDVPPnj01adIkf5fi1rt3b02YMEEjR45s8By1/T2rT25AxwgAAACAoHM87weP5z/s3+enSgAAABBMysrK9M0332jy5MkaNmxYkwxFJGnmzJmKjo72dxlueXl5uummm3TLLbf4uxRJBCMAAAAAglDRseMez/MPHfJPIQAAAAgqS5cuVZcuXVRQUKAZM2b4u5zTat++ve677z5/l+GWmJioiRMnyjAMf5ciiWAEAAAAQBAqLTzp8bzwyBE/VQIAAIBgkpmZKYfDoezsbLVp08bf5aCBCEYAAAAABJ2KolKP5yXHC/1UCQAAAICmhmAEAAAAQNCpLKn0eF52othPlQAAAABoaghGAAAAAAQdZ5np8byipNxPlQAAAABoaghGAAAAAAQds8Lzpo6OUoefKgEAAADQ1BCMAAAAAAg6ZqXV47mzzE+FAAAAAGhyCEYAAAAABB9HiMdTs5KPPgAAAABc+HQAAAAAIOiYTpvnDoe15oEAAABo1tLS0jRhwgR/l+F1r7zyitLT0xttvQ4dOmjOnDmnfT0vL08JCQk6ePBgo9VUG4IRAAAAAMHHDJUkWSuLXM8dtloGAwAAAE3P3r17ZRjGKY9333231uPKyso0ZcoUTZ48WZIrtKhpnupHWlqaz88lMTFRo0aN0uOPP+7zteoi5MxDAAAAACDQhEmSQioL5AiJcgclAAAAQFNTUVEhm+30v8jz//7f/9NFF13kft6yZcta51uxYoWio6PVv39/SdLWrVvlcDgkSVu2bNFvfvMbffXVV4qNjZUkhYY2zv+Vb7/9dvXu3VszZ85UixYtGmXN06FjBAAAAEDwMVzBiOEskCSZBCMAAACogyVLlig1NVUxMTGy2+0aOXKk8vLyJEmmaapTp06aNWuWxzG7d++WxWLRt99+K0kqKCjQuHHjlJiYqNjYWF111VXauXOne3xWVpZ69OihBQsWqGPHjgoLC5NpmqetKT4+Xna73f04U5CxbNky3XDDDe7nCQkJ7mOrQ5XExETZ7XYlJCTo97//vVJSUhQREaEuXbpo7ty5HvNlZmZqyJAhmjVrlpKTkxUfH6977rlHFRUVHuOKi4s1evRoxcTEqF27dnrppZc8Xu/WrZvsdrvefvvtWutvDAQjAAAAAIKOWRWMyDhRtSfMb7UAAAA0S6YplRf551FLyHAm5eXlevLJJ7Vz506tXLlSOTk5yszMlCQZhqHRo0dr4cKFHscsWLBA/fv313nnnSfTNJWRkaHc3FytWbNG2dnZ6tWrl66++modO3bMfcw333yj5cuXa8WKFdqxY0etNd1www1KTEzU5ZdfrjfffPOM57B582alpqbW6XydTqfOPfdcLV++XHv27NGUKVP06KOPavny5R7jNmzYoG+//VYbNmzQ4sWLtWjRIi1atMhjzNNPP63U1FRt375d48eP1913360vv/zSY0zv3r21efPmOtXmS1xKCwAAAEDQMY1wSZIRUlz1nGAEAACgUVUUS9Na+2ftRw9JoVENOnT06NHu7Y4dO+rZZ59V7969dfLkSUVHR+v222/XlClT9Omnn6p3796qqKjQkiVLNHPmTEmuAGHXrl3Ky8tTWJjr/6CzZs3SypUr9eabb2rcuHGSXAHMq6++qoSEhNPWEh0drWeeeUaXX365LBaLVq1apeHDh2vx4sW69dZbazzm+PHjOn78uFq3rtv33mazaerUqe7nKSkp2rJli5YvX65hw4a597do0ULz5s2T1WrVBRdcoIyMDL333nsaO3ase8zgwYM1fvx4SdLDDz+s2bNna+PGjbrgggvcY9q0aaPt27fXqTZfIhgBAAAAEHSclqpLaUVUSJU/BSUAAABAbbZv366srCzt2LFDx44dk9PplCTt379fXbt2VXJysjIyMrRgwQL17t1bq1evVmlpqW6++WZJUnZ2tk6ePKn4+HiPeUtKStyX2pKk9u3b1xqKSFKrVq30wAMPuJ+npqYqPz9fM2bMOG0wUlJSIkkKD6/7/39ffPFF/fWvf9W+fftUUlKi8vJy9ejRw2PMRRddJKvV6n6enJysXbt2eYzp3r27e9swDNntdvdlyKpFRESouLi4zrX5CsEIAAAAgKDjsLo+CIbG2VR8VDItdIwAAAA0Klukq3PDX2s3QFFRkdLT05Wenq4lS5YoISFB+/fv16BBg1ReXu4ed8cdd2jUqFGaPXu2Fi5cqOHDhysy0rWm0+lUcnKyNm7ceMr855xzjns7KqphHS2XXXaZ/vrXv5729fj4eBmGofz8/DrNt3z5cj3wwAN6+umn1bdvX8XExGjmzJn65JNPPMb9/ObwhmG4Q6P6jDl27NgZA6HGQDACAAAAIKicOH5cpsX1oSza3kLHj0pOCx0jAAAAjcowGnw5K3/58ssvdeTIET311FNq27atJGnbtm2njBs8eLCioqI0f/58vfPOO/rggw/cr/Xq1Uu5ubkKCQlRhw4dvF7j9u3blZycfNrXQ0ND1bVrV+3Zs0fp6elnnG/z5s3q16+f+xJYkjw6W7xt9+7dSktL89n8dcXN1wEAAAAElR/2fePeTup0niTJYQ1TZUWFv0oCAABAAGjXrp1CQ0P13HPP6bvvvtOqVav05JNPnjLOarUqMzNTkyZNUqdOndS3b1/3awMGDFDfvn01ZMgQrV27Vnv37tWWLVv0hz/8ocaQpTaLFy/Wa6+9pi+++EJfffWVZs2apWeffVb33XdfrccNGjRIH374YZ3W6NSpk7Zt26a1a9fq66+/1uTJk7V169Z61VlXxcXFys7OrlNg42sEIwAAAACCypFDrks2GM4KtbvgQtdOw6Ijh/b7sSoAAAA0dQkJCVq0aJHeeOMNde3aVU899ZRmzZpV49gxY8aovLzc42btkuvyUWvWrNEVV1yh0aNH6/zzz9eIESO0d+9eJSUl1bumP/7xj0pNTdUvfvELLVu2TAsWLPC470hNxo4dqzVr1qigoOCM8991110aOnSohg8frj59+ujo0aMe3SPe9I9//EPt2rVT//79fTJ/fRimaZr+LqK+CgsLFRcXp4KCAsXGxvq7HAAAAABNyPq/vaKvt6QopOKkxrx4rf5y7ybJsKj/KEPdL/+Vv8sDAAAISqWlpcrJyVFKSkq9bvwdqD766COlpaXp+++/b1Dg4WvDhg1Tz549NWnSJH+X4ta7d29NmDBBI0eObPActf09q09uQMcIAAAAgKBSlF8oSbI4SxVis8nqKJMkHf8h159lAQAAIAiUlZXpm2++0eTJkzVs2LAmGYpI0syZMxUdHe3vMtzy8vJ000036ZZbbvF3KZIIRgAAAAAEmfKTRZIkw3QFIhan6+uJI/l+qwkAAADBYenSperSpYsKCgo0Y8YMf5dzWu3btz/jvUgaU2JioiZOnCjDMPxdiiSCEQAAAABBpqLIFYQYZqnrq9P1tbTgpN9qAgAAQHDIzMyUw+FQdna22rRp4+9y0EAEIwAAAACCSmWJw7VhVgckrq/lJ0v8VRIAAACAJoRgBAAAAEBQcVZUbRhVwYhcXytLKk5zBAAAAIDmhGAEAAAAQFAxy6uvW1xetcMVjDjKnP4pCAAAAECTQjACAAAAIKiYjhBJkmGp6hCp6hwxy/1VEQAAAICmhGAEAAAAQHBxuoIRWSslSYbFlYiYlVZ/VQQAAACgCSEYAQAAABBcnKGuryFVN2E3XJ0j1Z0kAAAAAJo3ghEAAAAAQcYVjFhspuup1RWQGE6CEQAAAHhKS0vThAkT/F2G173yyitKT09vtPU6dOigOXPmnPb1vLw8JSQk6ODBg41WU20IRgAAAAAEFzNMkmQJc92E3bA5PfYDAAAAgWLt2rW67LLLFBMTo4SEBP3mN79RTk5OrceUlZVpypQpmjx5siRXaGEYxmkfaWlpPj+PxMREjRo1So8//rjP16oLghEAAAAAQcU0XAFISISrQ8RSdWUt0wz1V0kAAADAaVVUVNS4/7vvvtONN96oq666Sjt27NDatWt15MgRDR06tNb5VqxYoejoaPXv31+StHXrVh0+fFiHDx/WihUrJElfffWVe99bb73l3RM6jdtvv11///vflZ+f3yjr1YZgBAAAAEBwMcIlSbYo11drWPVN1+kYAQAAQO2WLFmi1NRUxcTEyG63a+TIkcrLy5MkmaapTp06adasWR7H7N69WxaLRd9++60kqaCgQOPGjVNiYqJiY2N11VVXaefOne7xWVlZ6tGjhxYsWKCOHTsqLCxMpmmeUstnn30mh8OhP/7xjzrvvPPUq1cvPfTQQ9q5c+dpwxRJWrZsmW644Qb384SEBNntdtntdrVs2VKSq4PDbrcrISFBv//975WSkqKIiAh16dJFc+fO9ZgvMzNTQ4YM0axZs5ScnKz4+Hjdc889p9RQXFys0aNHKyYmRu3atdNLL73k8Xq3bt1kt9v19ttvn7b2xkIwAgAAACCoOC2uACQ8NlqSFBJlk/RTJwkAAAB8zzRNFVcU++VRU8hQV+Xl5XryySe1c+dOrVy5Ujk5OcrMzJQkGYah0aNHa+HChR7HLFiwQP3799d5550n0zSVkZGh3NxcrVmzRtnZ2erVq5euvvpqHTt2zH3MN998o+XLl2vFihXasWNHjbWkpqbKarVq4cKFcjgcKigo0Kuvvqr09HTZbLbTnsPmzZuVmppap/N1Op0699xztXz5cu3Zs0dTpkzRo48+quXLl3uM27Bhg7799ltt2LBBixcv1qJFi7Ro0SKPMU8//bRSU1O1fft2jR8/Xnfffbe+/PJLjzG9e/fW5s2b61SbL3H3QQAAAABBxWlxdYpEtTxHkhQWEylJMqv2AwAAwPdKKkvU57U+fln7k5GfKNIW2aBjR48e7d7u2LGjnn32WfXu3VsnT55UdHS0br/9dk2ZMkWffvqpevfurYqKCi1ZskQzZ86U5AoQdu3apby8PIWFuX4xZ9asWVq5cqXefPNNjRs3TpIrgHn11VeVkJBw2lo6dOigdevW6eabb9add94ph8Ohvn37as2aNac95vjx4zp+/Lhat25dp/O12WyaOnWq+3lKSoq2bNmi5cuXa9iwYe79LVq00Lx582S1WnXBBRcoIyND7733nsaOHeseM3jwYI0fP16S9PDDD2v27NnauHGjLrjgAveYNm3aaPv27XWqzZfoGAEAAAAQNCorKuSwuj6AnpOYJEmKiIuVJDkNghEAAADUbvv27brxxhvVvn17xcTEuG9Mvn//fklScnKyMjIytGDBAknS6tWrVVpaqptvvlmSlJ2drZMnTyo+Pl7R0dHuR05OjvtSW5LUvn37WkMRScrNzdUdd9yh3/3ud9q6das2bdqk0NBQ3XTTTaftiikpKZEkhYfX/f++L774olJTU5WQkKDo6Gi9/PLL7vOtdtFFF8lqtbqfJycnuy8xVq179+7ubcMwZLfbTxkTERGh4uLiOtfmK3SMAAAAAAga+T/mSobrA1urc8+VJMXEx0uSnFYupQUAANBYIkIi9MnIT/y2dkMUFRUpPT1d6enpWrJkiRISErR//34NGjRI5eXl7nF33HGHRo0apdmzZ2vhwoUaPny4IiNdHSpOp1PJycnauHHjKfOfc8457u2oqKgz1vP8888rNjZWM2bMcO9bsmSJ2rZtq08++USXXXbZKcfEx8fLMIw63+B8+fLleuCBB/T000+rb9++iomJ0cyZM/XJJ55/dj+/dJdhGHI6nfUec+zYsTMGQo2BYAQAAABA0Phh3z73tr1dR0lSy2S7pDI5rWEqKSpSRB0+hAIAAODsGIbR4MtZ+cuXX36pI0eO6KmnnlLbtm0lSdu2bTtl3ODBgxUVFaX58+frnXfe0QcffOB+rVevXsrNzVVISIg6dOhwVvUUFxd7dGlIcj//eeBQLTQ0VF27dtWePXuUnp5+xjU2b96sfv36uS+BJcmjs8Xbdu/e7e7C8ScupQUAAAAgaBzPOyxJsjjKFFp1+YDEdh3cr/9wIMcfZQEAACAAtGvXTqGhoXruuef03XffadWqVXryySdPGWe1WpWZmalJkyapU6dO6tu3r/u1AQMGqG/fvhoyZIjWrl2rvXv3asuWLfrDH/5QY8hSm4yMDG3dulVPPPGE/vOf/+izzz7T7bffrvbt26tnz56nPW7QoEH68MMP67RGp06dtG3bNq1du1Zff/21Jk+erK1bt9arzroqLi5WdnZ2nQIbXyMYAQAAABA0Tvx4VJJkcZS698XFJ8pwVkqSjh484Je6AAAA0PQlJCRo0aJFeuONN9S1a1c99dRTmjVrVo1jx4wZo/Lyco+btUuuTpk1a9boiiuu0OjRo3X++edrxIgR2rt3r5KSkupVz1VXXaXXXntNK1euVM+ePXXNNdcoLCxM7777riIiTn+5sLFjx2rNmjUqKCg44xp33XWXhg4dquHDh6tPnz46evSoR/eIN/3jH/9Qu3bt1L9/f5/MXx+Gebq7tDRQZWWlsrKy9Pe//125ublKTk5WZmam/vCHP8hiceUwpmlq6tSpeumll5Sfn68+ffro+eef10UXXVSnNQoLCxUXF6eCggLFxsZ6s3wAAAAAAeytZ2bq8NeXKrTsB41deIt7/4tjVslhi9aFV36vq265zY8VAgAABKfS0lLl5OQoJSWlXjf+DlQfffSR0tLS9P3339c78GgMw4YNU8+ePTVp0iR/l+LWu3dvTZgwQSNHjmzwHLX9PatPbuD1jpE///nPevHFFzVv3jx98cUXmjFjhmbOnKnnnnvOPWbGjBl65plnNG/ePG3dulV2u10DBw7UiRMnvF0OAAAAgGak/ESJJMlwlnnst1Q9LzpWt5tQAgAAADUpKyvTN998o8mTJ2vYsGFNMhSRpJkzZyo6OtrfZbjl5eXppptu0i233HLmwY3A68HIxx9/rBtvvFEZGRnq0KGDbrrpJqWnp7uvn2aapubMmaPHHntMQ4cO1cUXX6zFixeruLhYr732mrfLAQAAANCMVBaXS5IM/SwYMV2X1iotLGr0mgAAABA8li5dqi5duqigoEAzZszwdzmn1b59e913333+LsMtMTFREydOlGEY/i5Fkg+CkV/+8pd677339PXXX0uSdu7cqQ8//FCDBw+WJOXk5Cg3N9fjBithYWG68sortWXLlhrnLCsrU2FhoccDAAAAAH7OUep0bZiewYhR9byiqOznhwAAAAB1lpmZKYfDoezsbLVp08bf5aCBQrw94cMPP6yCggJdcMEFslqtcjgc+tOf/uRukcnNzZWkU1qMkpKStG/fvhrnnD59uqZOnertUgEAAAAEGWdF1YbxswCkKhipLK1s3IIAAAAANDle7xh5/fXXtWTJEr322mv67LPPtHjxYs2aNUuLFy/2GPfzlhnTNE/bRjNp0iQVFBS4HwcOHPB22QAAAACCgFlR9RHHKP/ZK65gxFlmNm5BAAAAAJocr3eM/P73v9cjjzyiESNGSJK6deumffv2afr06frd734nu90uydU5kpyc7D4uLy/vtDeqCQsLU1hYmLdLBQAAABBsHCGSVTIsFR67DYsrKHFWNI1rGgMAAADwH693jBQXF8ti8ZzWarXK6XRd6zclJUV2u13r1693v15eXq5NmzapX79+3i4HAAAAQDNiOm2uDavD84XqDpJKa+MWBAAAAKDJ8XrHyPXXX68//elPateunS666CJt375dzzzzjEaPHi3JdQmtCRMmaNq0aercubM6d+6sadOmKTIyUiNHjvR2OQAAAACaEzPU9dX2s2DEWnVvEYfXPwIBAAAACDBe/1Tw3HPPafLkyRo/frzy8vLUunVr3XnnnZoyZYp7zMSJE1VSUqLx48crPz9fffr00bp16xQTE+PtcgAAAAA0J1XBiNX2s/0hrqDE3VECAAAAoNny+qW0YmJiNGfOHO3bt08lJSX69ttv9cc//lGhoaHuMYZhKCsrS4cPH1Zpaak2bdqkiy++2NulAAAAAGh2XPcmtIR7ftSxhFTfdD1UAAAAQLW0tDRNmDDB32X4xeTJkzVu3LhGW88wDK1cufK0r+/atUvnnnuuioqKfF6L14MRAAAAAPAX0wiXJNkiPDtDLGFVN103wxq7JAAAAKBBSktLlZmZqW7duikkJERDhgw5Zczhw4c1cuRIdenSRRaLpc4hzw8//KC5c+fq0UcfleQKLWp7ZGZmeu/ETqNbt27q3bu3Zs+e7fO1CEYAAAAABA3TcAUftugIj/3W8KqrCBsEIwAAAGhaKioqatzvcDgUERGh+++/XwMGDKhxTFlZmRISEvTYY4/pkksuqfOar7zyivr27asOHTpIcgUs1Y85c+YoNjbWY9/cuXPrfV4Ncfvtt2v+/PlyOBxnHnwWCEYAAAAABA3T4go+IuI8718YGuXaX91RAgAAANRkyZIlSk1NVUxMjOx2u0aOHKm8vDxJkmma6tSpk2bNmuVxzO7du2WxWPTtt99KkgoKCjRu3DglJiYqNjZWV111lXbu3Oken5WVpR49emjBggXq2LGjwsLCZJqmfi4qKkrz58/X2LFjZbfba6y3Q4cOmjt3rm677TbFxcXV+TyXLVumG264wf3cbre7H3FxcTIMw/3cZrPprrvu0rnnnqvIyEh169ZNS5cu9ZgvLS1N999/vyZOnKiWLVvKbrcrKyvrlHWPHDmiX//614qMjFTnzp21atUqj9cHDRqko0ePatOmTXU+l4YgGAEAAAAQNBxWV/AREx/vsT8sJlLSTx0lAAAA8C3TNOUsLvbLo6aQoa7Ky8v15JNPaufOnVq5cqVycnLcl5EyDEOjR4/WwoULPY5ZsGCB+vfvr/POO0+maSojI0O5ublas2aNsrOz1atXL1199dU6duyY+5hvvvlGy5cv14oVK7Rjx44G19sQ+fn52r17t1JTU+s0vrS0VJdeeqlWr16t3bt3a9y4cRo1apQ++eQTj3GLFy9WVFSUPvnkE82YMUNPPPGE1q9f7zFm6tSpGjZsmD7//HMNHjxYv/3tbz2+L6Ghobrkkku0efPmsz/RWoT4dHYAAAAAaCTlpaVyVgUjcfYkj9eiWp4jSXJa6BgBAABoDGZJib7qdalf1u7yWbaMyMgGHTt69Gj3dseOHfXss8+qd+/eOnnypKKjo3X77bdrypQp+vTTT9W7d29VVFRoyZIlmjlzpiRpw4YN2rVrl/Ly8hQW5vqlnFmzZmnlypV688033Tc7Ly8v16uvvqqEhISzPNv627dvn0zTVOvWres0vk2bNnrooYfcz++77z69++67euONN9SnTx/3/u7du+vxxx+XJHXu3Fnz5s3Te++9p4EDB7rHZGZm6pZbbpEkTZs2Tc8995w+/fRTXXPNNR7r7d2792xO8YzoGAEAAAAQFH44sM+9bW/bweO12MREST91lAAAAAA12b59u2688Ua1b99eMTExSktLkyTt379fkpScnKyMjAwtWLBAkrR69WqVlpbq5ptvliRlZ2fr5MmTio+PV3R0tPuRk5PjvtSWJLVv394voYgklZSUSJLCw+v2f2OHw6E//elP6t69u/u81q1b5/6eVOvevbvH8+TkZPdlyGoaExUVpZiYmFPGREREqLi4uM7n0xB0jAAAAAAICkcPHpBkkUynWto9f/stvvW5ko7ItIQo/8dctUio+RrNAAAA8A4jIkJdPsv229oNUVRUpPT0dKWnp2vJkiVKSEjQ/v37NWjQIJWXl7vH3XHHHRo1apRmz56thQsXavjw4Yqs6lBxOp1KTk7Wxo0bT5n/nHPOcW9HRUU1qEZvaNWqlSTXJbXqEs48/fTTmj17tubMmaNu3bopKipKEyZM8PieSJLNZvN4bhiGnE5nvcccO3ZM5513Xp3PpyEIRgAAAAAEheN5P0hKltVRqpCffeBKatte0hFJ0g/79xGMAAAA+JhhGA2+nJW/fPnllzpy5IieeuoptW3bVpK0bdu2U8YNHjzYfWP0d955Rx988IH7tV69eik3N1chISHq0KFDY5VeL+edd55iY2O1Z88enX/++Wccv3nzZt1444269dZbJbnCn//85z+68MILfVLf7t27ddNNN/lk7mpcSgsAAABAUDh5LF+SZHGWnfJaVGycLA7Xb7TlHzrUqHUBAAAgMLRr106hoaF67rnn9N1332nVqlV68sknTxlntVqVmZmpSZMmqVOnTurbt6/7tQEDBqhv374aMmSI1q5dq71792rLli36wx/+UGPIciZ79uzRjh07dOzYMRUUFGjHjh2n3Ky9et/Jkyf1448/aseOHdqzZ89p57RYLBowYIA+/PDDOtXQqVMnrV+/Xlu2bNEXX3yhO++8U7m5ufU+l7rYu3evDh48qAEDBvhk/moEIwAAAACCQmnBCUmS4Syt8XVL1f7CI0carSYAAAAEjoSEBC1atEhvvPGGunbtqqeeekqzZs2qceyYMWNUXl7ucbN2ydUps2bNGl1xxRUaPXq0zj//fI0YMUJ79+5VUlJSvWsaPHiwevbsqX/+85/auHGjevbsqZ49e3qMqd6XnZ2t1157TT179tTgwYNrnXfcuHFatmzZKZexqsnkyZPVq1cvDRo0SGlpabLb7RoyZEi9z6Uuli5dqvT0dLVv394n81czTNM0fbqCDxQWFiouLk4FBQWKjY31dzkAAAAAmoClU7J0LO8KhZXs1R2LR5/y+su3v67ysAS16bJdQx74Xz9UCAAAELxKS0uVk5OjlJSUOt/UO5B99NFHSktL0/fff9+gwMPfTNPUZZddpgkTJuiWW27xdzmSpLKyMnXu3FlLly7V5ZdfXuOY2v6e1Sc3oGMEAAAAQFCoLK6QJBnmqZfSkn7qJCk7UdxoNQEAACC4lJWV6ZtvvtHkyZM1bNiwgAxFJFdny0svvaTKykp/l+K2b98+PfbYY6cNRbyJm68DAAAACAqOsurLAJwmGKnaX1FS3kgVAQAAINgsXbpUY8aMUY8ePfTqq6/6u5yzcskll+iSSy7xdxlu559/fp1uBu8NdIwAAAAACApmpeHaMGoORlTVSeIodTRSRQAAAAg2mZmZcjgcys7OVps2bfxdDhqIYAQAAABAUDArrJIkw6ioeUBVYOI8TW4CAAAAoHkgGAEAAAAQHJxVVwq21hyMGIbrElpmJR+DAAAAgOaMTwQAAAAAgoLpDHVtWGu+gaRpqQpMHNZGqggAAABAU0QwAgAAACA4mK5gxLCZNb5sVAcmDltjVQQAAACgCSIYAQAAABAcqoIRS2jNLxshDo9xAAAAAJonghEAAAAAwcEIkySFhNd8qazqThKTYAQAAABo1ghGAAAAAAQFU+GSpJCosBpft4RVf/yp+XUAAAA0P2lpaZowYYK/y/CLyZMna9y4cY22nmEYWrly5Wlf37Vrl84991wVFRX5vBaCEQAAAABBwbS4Ao+w6MgaX7dFujpFTINgBAAAAE1faWmpMjMz1a1bN4WEhGjIkCG1jv/oo48UEhKiHj16nHHuH374QXPnztWjjz4qyRVa1PbIzMw8+xM6g27duql3796aPXu2z9ciGAEAAAAQFJwWV8dIZIvYGl+3VXWSmEZ4o9UEAAAAnElFRUWN+x0OhyIiInT//fdrwIABtc5RUFCg2267TVdffXWd1nzllVfUt29fdejQQZJ0+PBh92POnDmKjY312Dd37tx6nVND3X777Zo/f74cDodP1yEYAQAAABAUnFUdIzGtWtX4ekRcjKSfOksAAACAn1uyZIlSU1MVExMju92ukSNHKi8vT5JkmqY6deqkWbNmeRyze/duWSwWffvtt5JcIcW4ceOUmJio2NhYXXXVVdq5c6d7fFZWlnr06KEFCxaoY8eOCgsLk2map9QSFRWl+fPna+zYsbLb7bXWfeedd2rkyJHq27dvnc5z2bJluuGGG9zP7Xa7+xEXFyfDMNzPbTab7rrrLp177rmKjIxUt27dtHTpUo/50tLSdP/992vixIlq2bKl7Ha7srKyTln3yJEj+vWvf63IyEh17txZq1at8nh90KBBOnr0qDZt2lSn82goghEAAAAAAa+osEBOq+tSWfHJrWscEx3fQtJPnSUAAADwHdM0VVHm8MujppChrsrLy/Xkk09q586dWrlypXJyctyXkTIMQ6NHj9bChQs9jlmwYIH69++v8847T6ZpKiMjQ7m5uVqzZo2ys7PVq1cvXX311Tp27Jj7mG+++UbLly/XihUrtGPHjgbXK0kLFy7Ut99+q8cff7xO4/Pz87V7926lpqbWaXxpaakuvfRSrV69Wrt379a4ceM0atQoffLJJx7jFi9erKioKH3yySeaMWOGnnjiCa1fv95jzNSpUzVs2DB9/vnnGjx4sH772996fF9CQ0N1ySWXaPPmzXWqraFCfDo7AAAAADSCw/u+c28nte9Y45hzkly/ZeewhqmyokIhNluj1AYAANAcVZY79dL/+Pa3/k9n3NwrZQuzNujY0aNHu7c7duyoZ599Vr1799bJkycVHR2t22+/XVOmTNGnn36q3r17q6KiQkuWLNHMmTMlSRs2bNCuXbuUl5ensDBXp/KsWbO0cuVKvfnmm+6bnZeXl+vVV19VQkLCWZ3rf/7zHz3yyCPavHmzQkLq9uP+ffv2yTRNtW5d8y8U/VybNm300EMPuZ/fd999evfdd/XGG2+oT58+7v3du3d3hzOdO3fWvHnz9N5772ngwIHuMZmZmbrlllskSdOmTdNzzz2nTz/9VNdcc43Henv37q1TbQ1FxwgAAACAgHfs8EFJkuGsUFx8zZfSSji3rWvDsOjIof2NVRoAAAACyPbt23XjjTeqffv2iomJUVpamiRp/37X/x+Tk5OVkZGhBQsWSJJWr16t0tJS3XzzzZKk7OxsnTx5UvHx8YqOjnY/cnJy3JfakqT27dufdSjicDg0cuRITZ06Veeff36djyspKZEkhYfXrZPa4XDoT3/6k7p37+4+r3Xr1rm/J9W6d+/u8Tw5Odl9GbKaxkRFRSkmJuaUMRERESouLq7z+TQEHSMAAAAAAl5h3hFJ7WR1lJ12TOK5HSRzn2RYlPf9ftnbn9do9QEAADQ3IaEWjZt7pd/WboiioiKlp6crPT1dS5YsUUJCgvbv369BgwapvLzcPe6OO+7QqFGjNHv2bC1cuFDDhw9XZGSkJMnpdCo5OVkbN248Zf5zzjnHvR0VFdWgGv/biRMntG3bNm3fvl333nuve33TNBUSEqJ169bpqquuOuW4VlX35MvPz69TOPP0009r9uzZmjNnjrp166aoqChNmDDB43siSbafdWQbhiGn01nvMceOHdN55/n2/+oEIwAAAAACXlF+vqR2sjhLTzsmxGaT1VEmR0iEjv+Q23jFAQAANEOGYTT4clb+8uWXX+rIkSN66qmn1Latq9t427Ztp4wbPHiw+8bo77zzjj744AP3a7169VJubq5CQkLUoUMHn9YbGxurXbt2eex74YUX9P777+vNN99USkpKjcedd955io2N1Z49e+rUabJ582bdeOONuvXWWyW5wpf//Oc/uvDCC8/+JGqwe/du3XTTTT6ZuxqX0gIAAAAQ8MoKiyRJhnn6jhFJsjhdr584ku/zmgAAABBY2rVrp9DQUD333HP67rvvtGrVKj355JOnjLNarcrMzNSkSZPUqVMn9e3b1/3agAED1LdvXw0ZMkRr167V3r17tWXLFv3hD3+oMWQ5kz179mjHjh06duyYCgoKtGPHDvfN2i0Wiy6++GKPR2JiosLDw3XxxReftivFYrFowIAB+vDDD+tUQ6dOnbR+/Xpt2bJFX3zxhe68807l5vrmF4327t2rgwcPasCAAT6ZvxrBCAAAAICAV1HkCjwM8/QdI5JkVHWUlBac9HlNAAAACCwJCQlatGiR3njjDXXt2lVPPfWUZs2aVePYMWPGqLy83ONm7ZKrU2bNmjW64oorNHr0aJ1//vkaMWKE9u7dq6SkpHrXNHjwYPXs2VP//Oc/tXHjRvXs2VM9e/Zs0Pn9t3HjxmnZsmWnXMaqJpMnT1avXr00aNAgpaWlyW63a8iQIWddQ02WLl2q9PR0tW/f3ifzVzNM0zR9uoIPFBYWKi4uTgUFBYqNjfV3OQAAAAD8bPEDj+pkyQCFF+/RmL/de9pxf/3dQpVFtFfLpM26ZerjjVghAABAcCstLVVOTo5SUlLqfFPvQPbRRx8pLS1N33//fYMCD38zTVOXXXaZJkyYoFtuucXf5UiSysrK1LlzZy1dulSXX355jWNq+3tWn9yAjhEAAAAAAc9ZUbVh1H4pLUOu1ytLKmodBwAAANSkrKxM33zzjSZPnqxhw4YFZCgiuTpbXnrpJVVWVvq7FLd9+/bpscceO20o4k3cfB0AAABAwDPLjapf+yo/w0BXMOIoO/MlAwAAAICfW7p0qcaMGaMePXro1Vdf9Xc5Z+WSSy7RJZdc4u8y3M4///w63QzeG+gYAQAAABDwTIfrd74Myxk6Qao6Sswz5CcAAABATTIzM+VwOJSdna02bdr4uxw0EMEIAAAAgMDnrGqGt9Z+KQDD4kpEzEqrrysCAAAA0EQRjAAAAAAIfM5Q19cQR+3jDFdHSXWHCQAAALzLNE1/l4Ag5q2/XwQjAAAAAIKAKxix2M7wQcnqCk4MJ8EIAACAN1mtro7c8nKuWQrfqf77Vf33raH4NAAAAAAg8JlhkiRLmFHrMMPmlJw/jQcAAIB3hISEKDIyUj/++KNsNpssFn4nH97ldDr1448/KjIyUiEhZxdtEIwAAAAACHim4Qo6QsJr/4hjCZVUKplmaCNUBQAA0HwYhqHk5GTl5ORo3759/i4HQcpisahdu3YyjNp/IepMCEYAAAAABD4jXJJki6q9E8QaZpVKJYmOEQAAAG8LDQ1V586duZwWfCY0NNQr3UgEIwAAAAACntPiCjrCY6NqHRcSZZMKfuowAQAAgHdZLBaFh4f7uwygVlzoDQAAAEDAc1pcH76jWraodVxYTKQkybTwYR0AAABorghGAAAAAAS0yooKOayuDpC4xIRax0bExUqSnAbBCAAAANBcEYwAAAAACGj5P+ZKhlWSFN+mba1jY+LjJUlOK5fSAgAAAJornwQjBw8e1K233qr4+HhFRkaqR48eys7Odr9umqaysrLUunVrRUREKC0tTf/+9799UQoAAACAIPfDvn3u7aS2KbWObZlsl+QKRkqKinxaFwAAAICmyevBSH5+vi6//HLZbDa988472rNnj55++mmdc8457jEzZszQM888o3nz5mnr1q2y2+0aOHCgTpw44e1yAAAAAAS543mHJUkWR5kiomq/+Xpiuw7u7R8O5PiyLAAAAABNVIi3J/zzn/+stm3bauHChe59HTp0cG+bpqk5c+boscce09ChQyVJixcvVlJSkl577TXdeeed3i4JAAAAQBA78eNRSfGyOErPODYuPlGGc49Mi1VHDx5Qhwsu9n2BAAAAAJoUr3eMrFq1Sqmpqbr55puVmJionj176uWXX3a/npOTo9zcXKWnp7v3hYWF6corr9SWLVtqnLOsrEyFhYUeDwAAAACQpJJCV+e5xTxzMBJis8nidI0ryPvRp3UBAAAAaJq8Hox89913mj9/vjp37qy1a9fqrrvu0v3336+//e1vkqTc3FxJUlJSksdxSUlJ7td+bvr06YqLi3M/2rat/YaKAAAAAJqP8hMlkiTDWVan8RaHa1zRsXyf1QQAAACg6fJ6MOJ0OtWrVy9NmzZNPXv21J133qmxY8dq/vz5HuMMw/B4bprmKfuqTZo0SQUFBe7HgQMHvF02AAAAgABVWVwuSTJUx2CkqrOktJCbrwMAAADNkdeDkeTkZHXt2tVj34UXXqj9+/dLkux2uySd0h2Sl5d3ShdJtbCwMMXGxno8AAAAAECSHKVO14ZZt2CkelxFUR3HAwAAAAgqXg9GLr/8cn311Vce+77++mu1b99ekpSSkiK73a7169e7Xy8vL9emTZvUr18/b5cDAAAAIMg5K6o2jLoFHUZVx0hlaaWPKgIAAADQlIV4e8IHHnhA/fr107Rp0zRs2DB9+umneumll/TSSy9Jcl1Ca8KECZo2bZo6d+6szp07a9q0aYqMjNTIkSO9XQ4AAACAIGdWWCSrJKO8jke4xjnLTJ/VBAAAAKDp8now8otf/EJvv/22Jk2apCeeeEIpKSmaM2eOfvvb37rHTJw4USUlJRo/frzy8/PVp08frVu3TjExMd4uBwAAAECwc4RIVsmwVJx5rOTuLHFW1HyPQwAAAADBzevBiCRdd911uu666077umEYysrKUlZWli+WBwAAANCMmE6ba8PqqNsB1QFKpdU3BQEAAABo0rx+jxEAAAAAaFRmqOurrW7BiLuzxOGT3xMDAAAA0MQRjAAAAAAIbFXBiNVWt+FGiCtAcXeaAAAAAGhWCEYAAAAABLgwSZIlvI4fb2zOqo1Q35QDAAAAoEkjGAEAAAAQ0EwjXJJki6hbB4g1tOqm62aYr0oCAAAA0IQRjAAAAAAIaKbhCjhs0RF1Gm+NqLq3iEEwAgAAADRHBCMAAAAAApppcQUcEXExdRpvi3SNr+40AQAAANC8EIwAAAAACGgOqyvgiImPr9P4sNhIST91mgAAAABoXghGAAAAAASs8tJSOauCkTh7Up2OiWrRQpLktNAxAgAAADRHBCMAAAAAAtYPB/a5t+1tO9TpmNjEVpIkh5WOEQAAAKA5IhgBAAAAELCOHjzg2jCdamlvXadjWia3cR1isang6BFflQYAAACgiSIYAQAAABCwjuf9IEmyOkoVYrPV6Zjk9h3d2z/s+84ndQEAAABoughGAAAAAASsk8fyJUkWZ1mdj4mKjZPFUS5JOnr4kE/qAgAAANB0EYwAAAAACFilBSckSYaztF7HVQcpJ45wKS0AAACguSEYAQAAABCwKopcAYdh1r1jRJIsVUFKcX6h12sCAAAA0LQRjAAAAAAIWJXFFZLqH4wYVR0jZSeLvV4TAAAAgKaNYAQAAABAwHKUOau26hmMyNUxUllUv+MAAAAABD6CEQAAAAABy6w0XBtGPQOOqg6TylKHlysCAAAA0NQRjAAAAAAIWGaFVZJkGBX1O7AqSHGWe7siAAAAAE0dwQgAAACAwOUMcX211i8YMQxXImJWGN6uCAAAAEATRzACAAAAIGCZzlDXhrWyfsdZqoIUR4iXKwIAAADQ1BGMAAAAAAhcpisYMWxmvQ4zqoMUh83bFQEAAABo4ghGAAAAAASuqmDEElq/w4wQh8fxAAAAAJoPghEAAAAAgcsIkySFhFvrd1hVh4lJMAIAAAA0OwQjAAAAAAKWqXBJUkhUWL2Os4RVfxSq33EAAAAAAh/BCAAAAICAZVpcwUZYdGS9jrNFujpFTINgBAAAAGhuCEYAAAAABCynxdUxEtkitl7H2ao6TEwj3Os1AQAAAGjaCEYAAAAABCxnVcdITKtW9TouIi5G0k8dJwAAAACaD4IRAAAAAAGpqLBATqvrkljxya3rdWx0fAtJP3WcAAAAAGg+CEYAAAAABKTD+75zbye171ivY89JskuSHNYwVVZUeLUuAAAAAE0bwQgAAACAgHTs8EFJkuGsUFx8/S6llXBuW9eGYdGRQ/u9XRoAAACAJoxgBAAAAEBAKsw7IkmyOsrqfWziuR0k0ylJyvueYAQAAABoTghGAAAAAASkovx8SZLFWVrvY0NsNnegcvyHXK/WBQAAAKBpIxgBAAAAEJDKCoslSYZZ/44RSbI4XcedOJLvtZoAAAAANH0EIwAAAAACUkWxK9gwzPp3jEiSUdVpUlpw0ms1AQAAAGj6CEYAAAAABCRHSaVro4EdI9WdJuUnS7xVEgAAAIAAQDACAAAAICA5yk3XhtHAYESu4ypLKrxVEgAAAIAAQDACAAAAIDBVVH+cKW/Y8VUdI44yp3fqAQAAABAQCEYAAAAABCSz0ipJMiwN7Pio6jQxG5irAAAAAAhMBCMAAAAAApLptLk2rJUNOt6wuBKR6oAFAAAAQPNAMAIAAAAgMFUHIyGOhh1vuDpNTEeIlwoCAAAAEAgIRgAAAAAEJjNMkmSxmQ073uoKVAwnwQgAAADQnBCMAAAAAAhQoZIkS5jRoKMNW9VN16sCFgAAAADNA8EIAAAAgIBkGuGSpJDwhnV8WEKr5jFDvVUSAAAAgABAMAIAAAAgMBmuTg9bVMM6Pqxh1Tddp2MEAAAAaE4IRgAAAAAEJGdVx0h4bFSDjg+Jct283TQIRgAAAIDmxOfByPTp02UYhiZMmODeZ5qmsrKy1Lp1a0VERCgtLU3//ve/fV0KAAAAgCDitLoCjaiWLRp0fFhMpCTJtIR7rSYAAAAATZ9Pg5GtW7fqpZdeUvfu3T32z5gxQ88884zmzZunrVu3ym63a+DAgTpx4oQvywEAAAAQJCorKuSsCjTiEhMaNEdEXKyknzpPAAAAADQPPgtGTp48qd/+9rd6+eWX1aLFT7/BZZqm5syZo8cee0xDhw7VxRdfrMWLF6u4uFivvfaar8oBAAAAEEQKjubJtLjuERLfpm2D5oiJj5f0U+cJAAAAgObBZ8HIPffco4yMDA0YMMBjf05OjnJzc5Wenu7eFxYWpiuvvFJbtmypca6ysjIVFhZ6PAAAAAA0X3n797q3k9qmNGiOlsl2Sa5gpKSoyBtlAQAAAAgAPglGli1bps8++0zTp08/5bXc3FxJUlJSksf+pKQk92s/N336dMXFxbkfbds27DfCAAAAAASHY4ddnx0sjjJFRDXs5uuJ7Tq4t384kOONsgAAAAAEAK8HIwcOHND//M//aMmSJQoPP/21eg3D8HhumuYp+6pNmjRJBQUF7seBAwe8WjMAAACAwHLi6FFJrmCkoeLiE2U4HZKkowf5jAEAAAA0FyHenjA7O1t5eXm69NJL3fscDoc++OADzZs3T1999ZUkV+dIcnKye0xeXt4pXSTVwsLCFBbGdX8BAAAAuJQUuC6vazFLGzxHiM0mi7NUDkuUCvJ+9FZpAAAAAJo4r3eMXH311dq1a5d27NjhfqSmpuq3v/2tduzYoY4dO8put2v9+vXuY8rLy7Vp0yb169fP2+UAAAAACEJlJ4olSYaz4cGI9FPHSdGx/LOuCQAAAEBg8HrHSExMjC6++GKPfVFRUYqPj3fvnzBhgqZNm6bOnTurc+fOmjZtmiIjIzVy5EhvlwMAAAAgCFUWVUiSDDX8UlrSTx0npYXcfB0AAABoLrwejNTFxIkTVVJSovHjxys/P199+vTRunXrFBMT449yAAAAAAQYR5nr3iAyzy4YqT6+ougs5wEAAAAQMBolGNm4caPHc8MwlJWVpaysrMZYHgAAAECQcZZXbRhnF2gYVR0jlaWVZ1kRAAAAgEDh9XuMAAAAAICvmRVVH2WM8toHnpHreGeZeZbzAAAAAAgUBCMAAAAAAo/DKkkyLBVnN09Vx4mzwjjbigAAAAAECIIRAAAAAIHHtLm+Wh1nN091sFJpPbt5AAAAAAQMghEAAAAAAcd0hro2bGcXjLg7ThyNcvtFAAAAAE0AwQgAAACAwGOGSZKstrObxghxBSum8ywnAgAAABAwCEYAAAAABB7DFYxYws/yI43NWbURenbzAAAAAAgYBCMAAAAAAo4pVzBiizi7Tg9raNVN16s6UAAAAAAEP4IRAAAAAAHHrOoYsUVHnNU81oiqe4sYBCMAAABAc0EwAgAAACDgmJZwSVJEXMxZzWOLdAUiphF+1jUBAAAACAwEIwAAAAACjtPiCjRi4uPPap6w2EhJP3WgAAAAAAh+BCMAAAAAAkplRYUcVleQEWdPOqu5olq0kCQ5LXSMAAAAAM0FwQgAAACAgJL3/V7JcH2UsbftcFZzxSa2kiR30AIAAAAg+BGMAAAAAAgoP35/wLVhOtXS3vqs5mqZ3MY1lcWmgqNHzrY0AAAAAAGAYAQAAABAQMk/fFiSZHWUKsRmO6u5ktt3dG//sO+7s5oLAAAAQGAgGAEAAAAQUE4ey5ckWZxlZz1XVGycLI5ySdLRw4fOej4AAAAATR/BCAAAAICAUlpwQpJkOEu9Ml91wHLiCJfSAgAAAJoDghEAAAAAAaWiyBVkGObZd4xIkqUqYCnOL/TKfAAAAACaNoIRAAAAAAGlsrhCkveCEaOqY6TsZLFX5gMAAADQtBGMAAAAAAgojjJn1ZaXghG5OkYqi7wzHwAAAICmjWAEAAAAQEAxKw3XhuGlIKOq86Sy1OGd+QAAAAA0aQQjAAAAAAKKWWGVJBlGhXcmNMolSc5y70wHAAAAoGkjGAEAAAAQWJwhrq9W7wYjZoXhnfkAAAAANGkEIwAAAAACiukMdW1YK70yn2GpahVxhHhlPgAAAABNG8EIAAAAgMBiuoIRw2Z6Zz5LVcDisHlnPgAAAABNGsEIAAAAgMBSFYxYQr0znWFz3XTdNAlGAAAAgOaAYAQAAABAYDHCJEkh4VbvTBdS1XlihnllPgAAAABNG8EIAAAAgIBiKlySFBLlnSDDGlb9sYhgBAAAAGgOCEYAAAAABBTT4gowwqIjvTJfSKTrElqmQTACAAAANAcEIwAAAAACitPi6hiJbBHrlflsVZ0nBCMAAABA80AwAgAAACCgOKs6RmJatfLKfOFxMZIksypwAQAAABDcCEYAAAAABIyiwgI5raGSpPjk1l6ZM7plC0k/BS4AAAAAghvBCAAAAICAcXjfd+7tpPYdvTLnOYlJkiSHNVyVFRVemRMAAABA00UwAgAAACBgHDt8UJJkOCsUF++dS2nFt2nr2jAsOpZ7yCtzAgAAAGi6CEYAAAAABIzCvCOSJKujzGtzJrVt797OPbDXa/MCAAAAaJoIRgAAAAAEjKL8fEmSxVnqtTlDw8NlcbjmK8j9wWvzAgAAAGiaCEYAAAAABIyywmJJkmF6r2NEkqxVwciJo0e9Oi8AAACApodgBAAAAEDAqCh2BSKG6b2OEUkynK55SwpOeHVeAAAAAE0PwQgAAACAgOEoqXRteLljpLoDpfxEiVfnBQAAAND0EIwAAAAACBiOctO1YXg7GHF1oFSWVHh1XgAAAABND8EIAAAAgMBRUf0RptzLE7uCFkeZ08vzAgAAAGhqCEYAAAAABAyz0ipJMixe7uwwXEGL6e28BQAAAECTQzACAAAAIGCYTptrw1rp3Ymrg5Gq4AUAAABA8CIYAQAAABA4qoOREIdXp63uQDEdIV6dFwAAAEDTQzACAAAAIHCYYZIki8307rxVHSiGk2AEAAAACHYEIwAAAAACSKgkyRJmeHVWw1Z10/Wq4AUAAABA8PJ6MDJ9+nT94he/UExMjBITEzVkyBB99dVXHmNM01RWVpZat26tiIgIpaWl6d///re3SwEAAAAQZEwjXJIUEu7dzg5L1RW6TDPUq/MCAAAAaHq8Hoxs2rRJ99xzj/71r39p/fr1qqysVHp6uoqKitxjZsyYoWeeeUbz5s3T1q1bZbfbNXDgQJ04ccLb5QAAAAAIJoaro8MW5d3ODmt49UcjOkYAAACAYOf1C+i+++67Hs8XLlyoxMREZWdn64orrpBpmpozZ44ee+wxDR06VJK0ePFiJSUl6bXXXtOdd97p7ZIAAAAABAlnVcdIeGyUV+cNiQqVCiTTIBgBAAAAgp3P7zFSUFAgSWrZsqUkKScnR7m5uUpPT3ePCQsL05VXXqktW7bUOEdZWZkKCws9HgAAAACaH6fVFVxEtWzh1XnDYiIlSaYl3KvzAgAAAGh6fBqMmKapBx98UL/85S918cUXS5Jyc3MlSUlJSR5jk5KS3K/93PTp0xUXF+d+tG3b1pdlAwAAAGiCKisq5KwKLuISE7w6d0RcrKSfOlIAAAAABC+fBiP33nuvPv/8cy1duvSU1wzD8HhumuYp+6pNmjRJBQUF7seBAwd8Ui8AAACApqvgaJ5Mi1WSFN/Gu78sFRMfL+mnjhQAAAAAwcvr9xipdt9992nVqlX64IMPdO6557r32+12Sa7OkeTkZPf+vLy8U7pIqoWFhSksjA8oAAAAQHOWt3+vezupbYpX526ZbJdUJqc1TCVFRYqI8u49TAAAAAA0HV7vGDFNU/fee6/eeustvf/++0pJ8fzAkpKSIrvdrvXr17v3lZeXa9OmTerXr5+3ywEAAAAQJI4ddl161+Io83pwkdiug3v7hwM5Xp0bAAAAQNPi9WDknnvu0ZIlS/Taa68pJiZGubm5ys3NVUlJiSTXJbQmTJigadOm6e2339bu3buVmZmpyMhIjRw50tvlAAAAAAgSJ44eleQKRrwtLj5RhtMhSTp6kEv3AgAAAMHM65fSmj9/viQpLS3NY//ChQuVmZkpSZo4caJKSko0fvx45efnq0+fPlq3bp1iYmK8XQ4AAACAIFFSUChJspilXp87xGaTxVkqhyVKBXk/en1+AAAAAE2H14MR0zTPOMYwDGVlZSkrK8vbywMAAAAIUmUniiVJhtP7wYjk6kRxhESp6Fi+T+YHAAAA0DR4/VJaAAAAAOALlUUVkiTD9P6ltKSfOlFKC4t8Mj8AAACApoFgBAAAAEBAcJQ5qrZ8E4yoKnCpKPLR/AAAAACaBIIRAAAAAAHBWe76ahjlPpnfqOoYqSyt9Mn8AAAAAJoGghEAAAAAAcGsqPr4Yviqo8MVuDjLznzfRAAAAACBi2AEAAAAQGBwWCVJpsVHHR1VgYuzwvDN/AAAAACaBIIRAAAAAIHBtEmSDKuPghGL6+buqrT6Zn4AAAAATQLBCAAAAICAYDpDXRs2R+0DG8ioDkYcIT6ZHwAAAEDTQDACAAAAIDCYYZIkq8030xshrsDFdPpoAQAAAABNAsEIAAAAgMBguIIRS7iPPsbYnFUbob6ZHwAAAECTQDACAAAAICCYcgUjtgjfdHRYQ6tuul7VmQIAAAAgOBGMAAAAAAgIZlXHiC06wifzWyOq7i1iEIwAAAAAwYxgBAAAAEBAMC3hkqSIuBifzG+LdAUiphHuk/kBAAAANA0EIwAAAAACgtPiCi5i4uN9Mn9YbKSknzpTAAAAAAQnghEAAAAATV5lRYUcVldgEWdP8skaUS1aSJKcFjpGAAAAgGBGMAIAAACgycv7fq9kuD6+2Nt28MkasYmtJMkdwAAAAAAITgQjAAAAAJq8H78/4NownWppb+2TNVomt3EtYbGp4OgRn6wBAAAAwP8IRgAAAAA0ecd/yJUkWR2lCrHZfLJGcvuO7u0f9n3nkzUAAAAA+B/BCAAAAIAm7+TRfEmSxVnmszWiYuNkcZRLko4ePuSzdQAAAAD4F8EIAAAAgCavpOCEJMlwlvp0nerg5cQRLqUFAAAABCuCEQAAAABNXkWRK7AwTN91jEiSpSp4Kc4v9Ok6AAAAAPyHYAQAAABAk1dR7LrEla+DEaOqY6TsZLFP1wEAAADgPwQjAAAAAJo8Z5mzasvHwYhcHSOVRb5dBwAAAID/EIwAAAAAaPLMCsO1Yfg4sKjqSKksdfh2HQAAAAB+QzACAAAAoMkzK62SJMOo8O1ChuuSXc5y3y4DAAAAwH8IRgAAAAA0eaYjxLVhbZxgxN2hAgAAACDoEIwAAAAAaPIMp821Ya307TqWqlaR6iAGAAAAQNAhGAEAAADQ5JlmqCTJsJm+XchSFbw4bL5dBwAAAIDfEIwAAAAAaPrMMEmSJdS3yxg2103XTZNgBAAAAAhWBCMAAAAAmj7DFYyEhFt9u0xIVUdKVRADAAAAIPgQjAAAAABo8kxVBSNRvg0srGHVH5EIRgAAAIBgRTACAAAAoMkzLeGSpLDoSJ+uExLpuoSWaRCMAAAAAMGKYAQAAABAk+e0uIKKyBaxPl3HVtWRQjACAAAABC+CEQAAAABNnrOqYySmVSufrhMeFyPppw4VAAAAAMGHYAQAAABAk1ZUWCCnNVSSFJ/c2qdrRbdsIemnDhUAAAAAwYdgBAAAAECT9sOBfe7tpPYdfbrWOYlJkiSHNVyVFRU+XQsAAACAfxCMAAAAAGjSjh76XpJkOCsUF+/bS2nFt2nr2jAsOpZ7yKdrAQAAAPAPghEAAAAATVphXp4kyeoo8/laSW3bu7dzD+z1+XoAAAAAGh/BCAAAAIAmrejYcUmSxVnq87VCw8NlcbjWKcj9wefrAQAAAGh8BCMAAAAAmrSyE8WSJMP0fceIJFmrgpETR482ynoAAAAAGhfBCAAAAIAmrbzIFYgYpu87RiTJcLrWKyk40SjrAQAAAGhcBCMAAAAAmjRHaaVro5E6Rqo7UypOljTKegAAAAAaF8EIAAAAgCbNWWa6NozGCkZcnSkVJRWNsh4AAACAxkUwAgAAAKBJc1YaVVvljbSiK4BxljobaT0AAAAAjYlgBAAAAEDTVmmVJBmWRurgMFwBjIOGEQAAACAoEYwAAAAAaNocIa6v1srGWa8qGFGFtXHWAwAAANCo/BqMvPDCC0pJSVF4eLguvfRSbd682Z/lAAAAAGiKzFDX1xBHoyzn7kxxEIwAAAAAwchvwcjrr7+uCRMm6LHHHtP27dvVv39/XXvttdq/f7+/SgIAAADQBJlOVzBisZmNs2BVZ4rptDXOegAAAAAald+CkWeeeUZjxozRHXfcoQsvvFBz5sxR27ZtNX/+fH+VBAAAAKBJCpMkWcKMM4zzDsNWddP16k4VAAAAAEElxB+LlpeXKzs7W4888ojH/vT0dG3ZsuWU8WVlZSorK3M/Lyws9HmNgejdBS/p4KbG+bAIAAAANJaK0A6SpJDwxvn4YrFJKpMqQrrold+93ChrAgAAAPVilmnM3+71dxUByy/ByJEjR+RwOJSUlOSxPykpSbm5uaeMnz59uqZOndpY5QWs4uMFKo241N9lAAAAAD6RcH67Rlkn+txzdPJbyWGLlsMW3ShrAgAAAPVhrSz2dwkBzS/BSDXD8OxuME3zlH2SNGnSJD344IPu54WFhWrbtq3P6ws0Xa+8QiVH/8/fZQAAAABeF9c2UQNva5zfiLtxwgNa/cI8FeXlN8p6AAAAQH1ZQqySrvN3GQHLL8FIq1atZLVaT+kOycvLO6WLRJLCwsIUFhbWWOUFrAsu7aMLLu3j7zIAAACAgBZis2nI/zzg7zIAAAAA+Ihfbr4eGhqqSy+9VOvXr/fYv379evXr188fJQEAAAAAAAAAgGbAb5fSevDBBzVq1Cilpqaqb9++eumll7R//37ddddd/ioJAAAAAAAAAAAEOb8FI8OHD9fRo0f1xBNP6PDhw7r44ou1Zs0atW/f3l8lAQAAAAAAAACAIGeYpmn6u4j6KiwsVFxcnAoKChQbG+vvcgAAAAAAAAAAgB/VJzfwyz1GAAAAAAAAAAAA/IFgBAAAAAAAAAAANBsEIwAAAAAAAAAAoNkgGAEAAAAAAAAAAM0GwQgAAAAAAAAAAGg2CEYAAAAAAAAAAECzEeLvAhrCNE1JUmFhoZ8rAQAAAAAAAAAA/ladF1TnB7UJyGDkxIkTkqS2bdv6uRIAAAAAAAAAANBUnDhxQnFxcbWOMcy6xCdNjNPp1KFDhxQTEyPDMPxdDhDwCgsL1bZtWx04cECxsbH+LgdAkOM9B0Bj4f0GQGPiPQdAY+I9BziVaZo6ceKEWrduLYul9ruIBGTHiMVi0bnnnuvvMoCgExsbyz+mABoN7zkAGgvvNwAaE+85ABoT7zmApzN1ilTj5usAAAAAAAAAAKDZIBgBAAAAAAAAAADNBsEIAIWFhenxxx9XWFiYv0sB0AzwngOgsfB+A6Ax8Z4DoDHxngOcnYC8+ToAAAAAAAAAAEBD0DECAAAAAAAAAACaDYIRAAAAAAAAAADQbBCMAAAAAAAAAACAZoNgBAAAAAAAAAAANBsEI0AzlZ+fr1GjRikuLk5xcXEaNWqUjh8/Xufj77zzThmGoTlz5visRgDBob7vNxUVFXr44YfVrVs3RUVFqXXr1rrtttt06NChxisaQMB44YUXlJKSovDwcF166aXavHlzreM3bdqkSy+9VOHh4erYsaNefPHFRqoUQDCoz3vOW2+9pYEDByohIUGxsbHq27ev1q5d24jVAgh09f1/TrWPPvpIISEh6tGjh28LBAIYwQjQTI0cOVI7duzQu+++q3fffVc7duzQqFGj6nTsypUr9cknn6h169Y+rhJAMKjv+01xcbE+++wzTZ48WZ999pneeustff3117rhhhsasWoAgeD111/XhAkT9Nhjj2n79u3q37+/rr32Wu3fv7/G8Tk5ORo8eLD69++v7du369FHH9X999+vFStWNHLlAAJRfd9zPvjgAw0cOFBr1qxRdna2fvWrX+n666/X9u3bG7lyAIGovu851QoKCnTbbbfp6quvbqRKgcBkmKZp+rsIAI3riy++UNeuXfWvf/1Lffr0kST961//Ut++ffXll1+qS5cupz324MGD6tOnj9auXauMjAxNmDBBEyZMaKTKAQSas3m/+W9bt25V7969tW/fPrVr186XJQMIIH369FGvXr00f/58974LL7xQQ4YM0fTp008Z//DDD2vVqlX64osv3Pvuuusu7dy5Ux9//HGj1AwgcNX3PacmF110kYYPH64pU6b4qkwAQaKh7zkjRoxQ586dZbVatXLlSu3YsaMRqgUCDx0jQDP08ccfKy4uzv1DSkm67LLLFBcXpy1btpz2OKfTqVGjRun3v/+9LrroosYoFUCAa+j7zc8VFBTIMAydc845PqgSQCAqLy9Xdna20tPTPfanp6ef9v3l448/PmX8oEGDtG3bNlVUVPisVgCBryHvOT/ndDp14sQJtWzZ0hclAggiDX3PWbhwob799ls9/vjjvi4RCHgh/i4AQOPLzc1VYmLiKfsTExOVm5t72uP+/Oc/KyQkRPfff78vywMQRBr6fvPfSktL9cgjj2jkyJGKjY31dokAAtSRI0fkcDiUlJTksT8pKem07y+5ubk1jq+srNSRI0eUnJzss3oBBLaGvOf83NNPP62ioiINGzbMFyUCCCINec/5z3/+o0ceeUSbN29WSAg/8gXOhI4RIIhkZWXJMIxaH9u2bZMkGYZxyvGmada4X5Kys7M1d+5cLVq06LRjADQfvny/+W8VFRUaMWKEnE6nXnjhBa+fB4DA9/P3kjO9v9Q0vqb9AFCT+r7nVFu6dKmysrL0+uuv1/hLIwBQk7q+5zgcDo0cOVJTp07V+eef31jlAQGN+BAIIvfee69GjBhR65gOHTro888/1w8//HDKaz/++OMpv41QbfPmzcrLy/O4tr/D4dD//u//as6cOdq7d+9Z1Q4gsPjy/aZaRUWFhg0bppycHL3//vt0iwDw0KpVK1mt1lN+azIvL++07y92u73G8SEhIYqPj/dZrQACX0Pec6q9/vrrGjNmjN544w0NGDDAl2UCCBL1fc85ceKEtm3bpu3bt+vee++V5Lp8n2maCgkJ0bp163TVVVc1Su1AoCAYAYJIq1at1KpVqzOO69u3rwoKCvTpp5+qd+/ekqRPPvlEBQUF6tevX43HjBo16pT/xA8aNEijRo3S7bfffvbFAwgovny/kX4KRf7zn/9ow4YN/MASwClCQ0N16aWXav369fr1r3/t3r9+/XrdeOONNR7Tt29f/fOf//TYt27dOqWmpspms/m0XgCBrSHvOZKrU2T06NFaunSpMjIyGqNUAEGgvu85sbGx2rVrl8e+F154Qe+//77efPNNpaSk+LxmINAQjADN0IUXXqhrrrlGY8eO1V/+8hdJ0rhx43TdddepS5cu7nEXXHCBpk+frl//+teKj48/5QeTNptNdrvd4xgA+G8Neb+prKzUTTfdpM8++0yrV6+Ww+Fw/6ZUy5YtFRoa6pdzAdD0PPjggxo1apRSU1PVt29fvfTSS9q/f7/uuusuSdKkSZN08OBB/e1vf5Mk3XXXXZo3b54efPBBjR07Vh9//LFeeeUVLV261J+nASBA1Pc9Z+nSpbrttts0d+5cXXbZZe7/z0RERCguLs5v5wEgMNTnPcdisejiiy/2OD4xMVHh4eGn7AfgQjACNFN///vfdf/99ys9PV2SdMMNN2jevHkeY7766isVFBT4ozwAQaS+7zfff/+9Vq1aJUnq0aOHx7gNGzYoLS3N5zUDCAzDhw/X0aNH9cQTT+jw4cO6+OKLtWbNGrVv316SdPjwYe3fv989PiUlRWvWrNEDDzyg559/Xq1bt9azzz6r3/zmN/46BQABpL7vOX/5y19UWVmpe+65R/fcc497/+9+9zstWrSoscsHEGDq+54DoH4Ms/pugwAAAAAAAAAAAEHO4u8CAAAAAAAAAAAAGgvBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJoNghEAAAAAAAAAANBsEIwAAAAAAAAAAIBmg2AEAAAAAAAAAAA0GwQjAAAAAAAAAACg2SAYAQAAAAAAAAAAzQbBCAAAAAAAAAAAaDYIRgAAAAAAAAAAQLNBMAIAAAAAAAAAAJqN/w/iTU6o6atUFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 2000x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize_histograms(layers, C, Xdev, 'gradients')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6b7ff70",
   "metadata": {},
   "source": [
    "> Only the last two layers's biases are trained i.e. Linear and Batchnorm layer. It is partially training because the last layer's tanh activations and gradients are zero. Therefore only the last layers biases are changing due to the loss function's derivative"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce48e63c",
   "metadata": {},
   "source": [
    "## E02 \n",
    "> BatchNorm, unlike other normalization layers like LayerNorm/GroupNorm etc. has the big advantage that after training, the batchnorm gamma/beta can be \"folded into\" the weights of the preceeding Linear layers, effectively erasing the need to forward it at test time. Set up a small 3-layer MLP with batchnorms, train the network, then \"fold\" the batchnorm gamma/beta into the preceeding Linear layer's W,b by creating a new W2, b2 and erasing the batch norm. Verify that this gives the same forward pass during inference. i.e. we see that the batchnorm is there just for stabilizing the training, and can be thrown out after training is done! pretty cool."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e89b1938",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X, Y, epochs, bs = 32, context_length = 3, vocab_size = 27, embedding_size = 10, n_hidden = 100, print_at_every_epoch=10):\n",
    "    \n",
    "    C = torch.randn((vocab_size, embedding_size), generator=g)\n",
    "    \n",
    "    layers = [\n",
    "        nn.Linear(context_length * embedding_size, vocab_size), \n",
    "        nn.BatchNorm1d(vocab_size), \n",
    "        nn.Tanh()\n",
    "    ]\n",
    "\n",
    "    print('Total number of layers are : ', len(layers))\n",
    "    parameters = [C] + [p for layer in layers for p in layer.parameters()]\n",
    "    print('Total number of parameters are :', sum(p.nelement() for p in parameters))\n",
    "\n",
    "    \n",
    "    # set parameters to required grad\n",
    "    for p in parameters:\n",
    "        p.requires_grad = True\n",
    "\n",
    "    \n",
    "        \n",
    "    lossi = []\n",
    "    for i in range(epochs):\n",
    "        ix = torch.randint(0, len(X), (bs,),  generator=g)\n",
    "        \n",
    "        Xb, Yb = X[ix], Y[ix]\n",
    "        \n",
    "        emb = C[Xb]\n",
    "        x = emb.view(emb.shape[0], -1)\n",
    "        \n",
    "        for layer in layers:\n",
    "            x = layer(x)\n",
    "\n",
    "        loss = F.cross_entropy(x, Yb)\n",
    "   \n",
    "        for p in parameters:\n",
    "            p.grad = None\n",
    "            \n",
    "        loss.backward()\n",
    "        \n",
    "        lr = 0.1 if i <= 100_000 else 0.01 # lr decay\n",
    "        \n",
    "        for p in parameters:\n",
    "            p.data += -lr * p.grad\n",
    "        \n",
    "        if print_at_every_epoch: \n",
    "            if i % print_at_every_epoch == 0:\n",
    "                print(f'Epoch {i} Loss {loss.item()}')\n",
    "        \n",
    "        lossi.append(loss.item())\n",
    "        \n",
    "    return C, layers, lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "242b6d76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of layers are :  3\n",
      "Total number of parameters are : 1161\n",
      "Epoch 0 Loss 3.587299108505249\n",
      "Epoch 10 Loss 3.2293121814727783\n",
      "Epoch 20 Loss 3.0814695358276367\n",
      "Epoch 30 Loss 3.2448253631591797\n",
      "Epoch 40 Loss 3.101447582244873\n",
      "Epoch 50 Loss 3.429870367050171\n",
      "Epoch 60 Loss 3.221737861633301\n",
      "Epoch 70 Loss 3.0934414863586426\n",
      "Epoch 80 Loss 3.1333014965057373\n",
      "Epoch 90 Loss 3.0430803298950195\n",
      "Epoch 100 Loss 3.222014904022217\n",
      "Epoch 110 Loss 2.9680709838867188\n",
      "Epoch 120 Loss 3.0302188396453857\n",
      "Epoch 130 Loss 2.913753032684326\n",
      "Epoch 140 Loss 3.072633743286133\n",
      "Epoch 150 Loss 2.923525333404541\n",
      "Epoch 160 Loss 2.755462646484375\n",
      "Epoch 170 Loss 2.9250504970550537\n",
      "Epoch 180 Loss 3.050321578979492\n",
      "Epoch 190 Loss 2.914196491241455\n",
      "Epoch 200 Loss 3.108175277709961\n",
      "Epoch 210 Loss 2.7188565731048584\n",
      "Epoch 220 Loss 2.8650524616241455\n",
      "Epoch 230 Loss 2.879669189453125\n",
      "Epoch 240 Loss 2.9316511154174805\n",
      "Epoch 250 Loss 3.002821207046509\n",
      "Epoch 260 Loss 2.710664987564087\n",
      "Epoch 270 Loss 2.9430744647979736\n",
      "Epoch 280 Loss 3.0812506675720215\n",
      "Epoch 290 Loss 2.9047486782073975\n",
      "Epoch 300 Loss 2.9102120399475098\n",
      "Epoch 310 Loss 2.798466205596924\n",
      "Epoch 320 Loss 2.9486002922058105\n",
      "Epoch 330 Loss 2.7822418212890625\n",
      "Epoch 340 Loss 2.922684669494629\n",
      "Epoch 350 Loss 3.0438175201416016\n",
      "Epoch 360 Loss 2.863776206970215\n",
      "Epoch 370 Loss 2.712582588195801\n",
      "Epoch 380 Loss 2.9411137104034424\n",
      "Epoch 390 Loss 2.966948986053467\n",
      "Epoch 400 Loss 2.787348985671997\n",
      "Epoch 410 Loss 2.6891531944274902\n",
      "Epoch 420 Loss 2.789918899536133\n",
      "Epoch 430 Loss 2.846391439437866\n",
      "Epoch 440 Loss 2.705667018890381\n",
      "Epoch 450 Loss 2.8475072383880615\n",
      "Epoch 460 Loss 2.6863129138946533\n",
      "Epoch 470 Loss 2.7115252017974854\n",
      "Epoch 480 Loss 2.7697083950042725\n",
      "Epoch 490 Loss 2.8165030479431152\n",
      "Epoch 500 Loss 2.652785539627075\n",
      "Epoch 510 Loss 2.7844464778900146\n",
      "Epoch 520 Loss 2.5727062225341797\n",
      "Epoch 530 Loss 2.773362636566162\n",
      "Epoch 540 Loss 2.639413356781006\n",
      "Epoch 550 Loss 2.937549352645874\n",
      "Epoch 560 Loss 2.733950614929199\n",
      "Epoch 570 Loss 2.9230353832244873\n",
      "Epoch 580 Loss 2.7119667530059814\n",
      "Epoch 590 Loss 2.629343032836914\n",
      "Epoch 600 Loss 2.8231914043426514\n",
      "Epoch 610 Loss 2.896239995956421\n",
      "Epoch 620 Loss 2.754451274871826\n",
      "Epoch 630 Loss 2.8002326488494873\n",
      "Epoch 640 Loss 2.694410800933838\n",
      "Epoch 650 Loss 2.56418776512146\n",
      "Epoch 660 Loss 2.7440552711486816\n",
      "Epoch 670 Loss 2.738240957260132\n",
      "Epoch 680 Loss 2.872488260269165\n",
      "Epoch 690 Loss 2.8462133407592773\n",
      "Epoch 700 Loss 2.6969306468963623\n",
      "Epoch 710 Loss 2.8000383377075195\n",
      "Epoch 720 Loss 2.663419246673584\n",
      "Epoch 730 Loss 2.8632514476776123\n",
      "Epoch 740 Loss 2.5700666904449463\n",
      "Epoch 750 Loss 2.537867784500122\n",
      "Epoch 760 Loss 2.601379156112671\n",
      "Epoch 770 Loss 2.5540926456451416\n",
      "Epoch 780 Loss 2.5457212924957275\n",
      "Epoch 790 Loss 2.7192153930664062\n",
      "Epoch 800 Loss 2.7309796810150146\n",
      "Epoch 810 Loss 2.7865402698516846\n",
      "Epoch 820 Loss 2.7576744556427\n",
      "Epoch 830 Loss 2.7156918048858643\n",
      "Epoch 840 Loss 2.634010076522827\n",
      "Epoch 850 Loss 2.7768185138702393\n",
      "Epoch 860 Loss 2.6386663913726807\n",
      "Epoch 870 Loss 2.7992215156555176\n",
      "Epoch 880 Loss 2.6494650840759277\n",
      "Epoch 890 Loss 2.6424853801727295\n",
      "Epoch 900 Loss 2.85687255859375\n",
      "Epoch 910 Loss 2.843587636947632\n",
      "Epoch 920 Loss 2.7913031578063965\n",
      "Epoch 930 Loss 2.5635101795196533\n",
      "Epoch 940 Loss 2.5651516914367676\n",
      "Epoch 950 Loss 2.635497570037842\n",
      "Epoch 960 Loss 2.7413291931152344\n",
      "Epoch 970 Loss 2.7723095417022705\n",
      "Epoch 980 Loss 2.8175578117370605\n",
      "Epoch 990 Loss 2.7129404544830322\n"
     ]
    }
   ],
   "source": [
    "C, layers, losses = train(Xtr, Ytr, 1000, print_at_every_epoch=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36a8dfa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94efa642",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_logits(embedding, model, X):\n",
    "    emb = embedding[X]\n",
    "    logits = emb.view(emb.shape[0], -1)\n",
    "    for layer in model: logits = layer(logits) \n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d75e5a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.4 ms ± 212 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "o1 = compute_logits(C, layers, Xdev)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a960ec57",
   "metadata": {},
   "source": [
    "> Lets fold the batchnorm into the previous linear layer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7a0ddf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_logits_folded_bn(embedding, model, X):\n",
    "    emb = embedding[X]\n",
    "    logits = emb.view(emb.shape[0], -1)\n",
    "    \n",
    "    # linear layer weight and bias\n",
    "    w0 = model[0].weight.T\n",
    "    b0 = model[0].bias\n",
    "    \n",
    "    # batchnorm weight and bias\n",
    "    w1 = model[1].weight.T\n",
    "    b1 = model[1].bias\n",
    "    \n",
    "    # batchnorm running mean and std\n",
    "    running_mean = layers[1].running_mean\n",
    "    running_std = torch.sqrt(layers[1].running_var)\n",
    "    \n",
    "    # new weight and bias\n",
    "    w2 = (w1 * w0) / running_std\n",
    "    b2 = w1 * (b0 - running_mean) / running_std + b1\n",
    "    \n",
    "    \n",
    "    return F.tanh( logits @ w2 + b2 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b78f2a3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.37 ms ± 201 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "o2 = compute_logits_folded_bn(C, layers, Xdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "762b1fe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8510,  0.9600,  0.6935,  ..., -0.5150, -0.5210,  0.6189],\n",
       "        [-0.8332,  0.4823, -0.3241,  ..., -0.4695,  0.7627, -0.2997],\n",
       "        [-0.8381,  0.8900, -0.6281,  ..., -0.7169,  0.8849, -0.6847],\n",
       "        ...,\n",
       "        [ 0.9905,  0.9764, -0.9539,  ..., -0.9095, -0.9541, -0.9398],\n",
       "        [ 0.9542,  0.9854, -0.8611,  ..., -0.9231,  0.6759, -0.9285],\n",
       "        [ 0.9966, -0.8805, -0.8403,  ..., -0.9066,  0.3652, -0.8431]],\n",
       "       grad_fn=<TanhBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9270ac28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8547,  0.9598,  0.6821,  ..., -0.5114, -0.5483,  0.6069],\n",
       "        [-0.8373,  0.4803, -0.3395,  ..., -0.4656,  0.7448, -0.3119],\n",
       "        [-0.8420,  0.8895, -0.6376,  ..., -0.7149,  0.8754, -0.6903],\n",
       "        ...,\n",
       "        [ 0.9903,  0.9763, -0.9550,  ..., -0.9090, -0.9571, -0.9405],\n",
       "        [ 0.9530,  0.9853, -0.8646,  ..., -0.9228,  0.6529, -0.9294],\n",
       "        [ 0.9965, -0.8813, -0.8444,  ..., -0.9061,  0.3298, -0.8455]],\n",
       "       grad_fn=<TanhBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b1a4f04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0003, grad_fn=<MseLossBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nn.functional.mse_loss(o1, o2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e57852ea",
   "metadata": {},
   "source": [
    "they are close, the little difference might be due to epsilon added to the variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5accd2c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "zhnn",
   "language": "python",
   "name": "zhnn"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}