BERT_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_training.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "jvjFMVLGRXkG",
        "xOMDc5GRRc1g"
      ],
      "authorship_tag": "ABX9TyPm2PsyH6rsbfLSNsGpCjZC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d49fdf16fd84405aaab96044ab6e951d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b06f5c21c0b484393865ff78045a236",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1121f00a1f6249699363e94802365850",
              "IPY_MODEL_743c20d9f53a4e61a4739426af5f4051"
            ]
          }
        },
        "4b06f5c21c0b484393865ff78045a236": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1121f00a1f6249699363e94802365850": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_83657714473246f1b05e05fedefc81b6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e57d132b45694b2cb00183d12d9ad1d5"
          }
        },
        "743c20d9f53a4e61a4739426af5f4051": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7c8b361f79584d3fb62ca1a760d5d7ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 2.59MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d6be092e5836416ab029f475e7970ebf"
          }
        },
        "83657714473246f1b05e05fedefc81b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e57d132b45694b2cb00183d12d9ad1d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c8b361f79584d3fb62ca1a760d5d7ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d6be092e5836416ab029f475e7970ebf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lupis30puc/BERT_interpretation_with_RF/blob/main/BERT_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-HbrtITbQFIB"
      },
      "source": [
        "[Yelp dataset on kaggle](https://www.kaggle.com/yelp-dataset/yelp-dataset)\n",
        "\n",
        "12,993 samples from the Yelp Dataset Challenge 2020. \n",
        "Divided on train and validation subsets. \n",
        "\n",
        "Their corresponding sizes are: 10,394 train samples, 2,599 test samples.\n",
        "\n",
        "\n",
        "Reference notebooks:\n",
        "[BERT Fine-Tunning](http://mccormickml.com/2019/07/22/BERT-fine-tuning/#41-bertforsequenceclassification)\n",
        "[Colab: BERT for sequential classification](https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX#scrollTo=DEfSbAA4QHas)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0SEJb-ieoMjd"
      },
      "source": [
        "With this notebook the model was run several times to obatain the best parameters among the following options:\n",
        "\n",
        "**max length = 128, 99** --representing is the 95 and 90 percentile of the dataset length respectively \n",
        "\n",
        "**batch size = 32, 16**\n",
        "\n",
        "**epochs = 10** and later on **3**\n",
        "\n",
        " **According to the made experiments the best parameters are max_length= 128, batch_size=32 and epochs = 3.**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jvjFMVLGRXkG"
      },
      "source": [
        "## Set up"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vEtLBK3KQmal",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "299b3a9b-aa34-4ee1-f2dc-78669189f7e2"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/87/ef312eef26f5cecd8b17ae9654cdd8d1fae1eb6dbd87257d6d73c128a4d0/transformers-4.3.2-py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 6.1MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 55.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 50.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=fdba594ce18b4900f97ebce8b0770d3d0801735d6eeb01cdc123150d3d77630b\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xOMDc5GRRc1g"
      },
      "source": [
        "## GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CfMcTnCvRXRG",
        "outputId": "c995f1f5-d69c-4897-aa28-d15e39537d15"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6rVlou0fRmHQ",
        "outputId": "fe04c939-7cc3-43c0-e577-a958208872a7"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FAodVLlMQoGJ"
      },
      "source": [
        "## Loading the pre-processed datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DO883dUWQn1r",
        "outputId": "10b20628-df4a-4c2f-f7fa-e75748242ecf"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "juAuzLM8RBPh"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "sns.set()\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lby1yJAZNGSM"
      },
      "source": [
        "# Loading the train and test datasets obatined on the preprocessing notebook\n",
        "train_df = pd.read_pickle('/content/drive/MyDrive/Yelp/sample_train_10394.pkl')\n",
        "validation_df = pd.read_pickle('/content/drive/MyDrive/Yelp/sample_test_2599.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ekDCH2DZCnxg",
        "outputId": "e3b8aa48-402c-4e34-fac3-6a09201e4da5"
      },
      "source": [
        "print(train_df.shape, validation_df.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(10394, 5) (2599, 5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ljRADMeRtON"
      },
      "source": [
        "Extract the cleaned texts and labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tz8r7hu4RxWs"
      },
      "source": [
        "# Get the lists of reviews and their labels.\n",
        "train_x = train_df.clean_text.values\n",
        "train_y = train_df.label.values\n",
        "\n",
        "val_x = validation_df.clean_text.values\n",
        "val_y = validation_df.label.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "USm3DlFeR3qb"
      },
      "source": [
        "# BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ngPhB59wR6m9"
      },
      "source": [
        "### BERT Tokenizer to create input ids and masks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "d49fdf16fd84405aaab96044ab6e951d",
            "4b06f5c21c0b484393865ff78045a236",
            "1121f00a1f6249699363e94802365850",
            "743c20d9f53a4e61a4739426af5f4051",
            "83657714473246f1b05e05fedefc81b6",
            "e57d132b45694b2cb00183d12d9ad1d5",
            "7c8b361f79584d3fb62ca1a760d5d7ea",
            "d6be092e5836416ab029f475e7970ebf"
          ]
        },
        "id": "ZCzTuzrPR3GY",
        "outputId": "762727fa-3d93-446e-a59e-a1895998326b"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d49fdf16fd84405aaab96044ab6e951d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fS0J2azYYZhe"
      },
      "source": [
        "def get_inputs_ids_mask(reviews, labels):\n",
        "  %%time\n",
        "  # Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "  input_ids = []\n",
        "  attention_masks = []\n",
        "\n",
        "  # For every sentence...\n",
        "  for sent in reviews:\n",
        "      # `encode_plus` will:\n",
        "      #   (1) Tokenize the sentence.\n",
        "      #   (2) Prepend the `[CLS]` token to the start.\n",
        "      #   (3) Append the `[SEP]` token to the end.\n",
        "      #   (4) Map tokens to their IDs.\n",
        "      #   (5) Pad or truncate the sentence to `max_length`\n",
        "      #   (6) Create attention masks for [PAD] tokens.\n",
        "      encoded_dict = tokenizer.encode_plus(\n",
        "                          sent,                      # Sentence to encode.\n",
        "                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                          max_length = 128, # SETTING THE MAX TO 128 tokens --> 95 PERCENTILE\n",
        "                          padding = 'max_length',\n",
        "                          return_attention_mask = True,   # Construct attn. masks.\n",
        "                          return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                          truncation = True\n",
        "                    )\n",
        "      \n",
        "      # Add the encoded sentence to the list.    \n",
        "      input_ids.append(encoded_dict['input_ids'])\n",
        "      \n",
        "      # And its attention mask (simply differentiates padding from non-padding).\n",
        "      attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "  # Convert the lists into tensors.\n",
        "  input_ids = torch.cat(input_ids, dim=0)\n",
        "  attention_masks = torch.cat(attention_masks, dim=0)\n",
        "  labels = torch.tensor(labels)\n",
        "\n",
        "  # Print sentence 0, now as a list of IDs.\n",
        "  print('Original: ', reviews[0])\n",
        "  print('Token IDs:', input_ids[0])\n",
        "  return input_ids, attention_masks, labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qNMaRwuftOoF",
        "outputId": "f59b6212-2a83-4044-fd5a-a690293315a6"
      },
      "source": [
        "train_ii, train_masks, train_labels = get_inputs_ids_mask(train_x, train_y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3 µs, sys: 0 ns, total: 3 µs\n",
            "Wall time: 6.68 µs\n",
            "Original:  second best pool las vegas pay commit day drink overpriced drinks hits blast speakers won let bring alcohol nice check pools fun creative kids especially adults water slide massage spa flamingo pretty outstanding\n",
            "Token IDs: tensor([  101,  2117,  2190,  4770,  5869,  7136,  3477, 10797,  2154,  4392,\n",
            "         2058, 18098,  6610,  2094,  8974,  4978,  8479,  7492,  2180,  2292,\n",
            "         3288,  6544,  3835,  4638, 12679,  4569,  5541,  4268,  2926,  6001,\n",
            "         2300,  7358, 21881, 12403, 19091,  2080,  3492,  5151,   102,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bmw0HmuwtVuy",
        "outputId": "5a31ffb0-8ea3-4878-8785-d99f118c10aa"
      },
      "source": [
        "val_ii, val_masks, val_labels = get_inputs_ids_mask(val_x, val_y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3 µs, sys: 0 ns, total: 3 µs\n",
            "Wall time: 6.44 µs\n",
            "Original:  positives good location staff reasonably friendly negatives dry manis pedis advertise website update site base coat coat blobbed attention coat cover color nail nails filed totally random lenths shapes shorter end fingers longer pedi woman wasn paying attention ended filing skin bled cute tea menu offer definitely expensive getting answer phone pedi station isn comfortable slightly padded bench notes staff looks surprised sure come ask work\n",
            "Token IDs: tensor([  101,  3893,  2015,  2204,  3295,  3095, 16286,  5379,  4997,  2015,\n",
            "         4318, 23624,  2015, 21877, 10521,  4748, 16874,  5562,  4037, 10651,\n",
            "         2609,  2918,  5435,  5435,  1038,  4135, 15499,  3086,  5435,  3104,\n",
            "         3609, 13774, 10063,  6406,  6135,  6721, 15307,  7898, 10466,  7820,\n",
            "         2203,  3093,  2936, 21877,  4305,  2450,  2347,  7079,  3086,  3092,\n",
            "        15242,  3096, 23919, 10140,  5572, 12183,  3749,  5791,  6450,  2893,\n",
            "         3437,  3042, 21877,  4305,  2276,  3475,  6625,  3621, 20633,  6847,\n",
            "         3964,  3095,  3504,  4527,  2469,  2272,  3198,  2147,   102,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ASzm3Yafbv_m"
      },
      "source": [
        "TO SAVE THE RESULTS"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jclOoLjtQ0Sx"
      },
      "source": [
        "# before creating the Tensor Datasets, I want to save the datasets input ids as lists\n",
        "# this later becomes the input for the random forest model\n",
        "import pickle \n",
        "\n",
        "train_tensor_list = train_ii.tolist()\n",
        "with open('/content/drive/MyDrive/Yelp/model_128_/train_ids_128.pkl', 'wb') as f:\n",
        "  pickle.dump(train_tensor_list, f)\n",
        "\n",
        "test_tensor_list = val_ii.tolist()\n",
        "with open('/content/drive/MyDrive/Yelp/model_128_/test_ids_128.pkl', 'wb') as v:\n",
        "  pickle.dump(test_tensor_list, v)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7ECdewLGZ2aE"
      },
      "source": [
        "### Creating tensor Datasets and DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9m8grhVtzg8"
      },
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "train_dataset = TensorDataset(train_ii, train_masks, train_labels)\n",
        "val_dataset = TensorDataset(val_ii, val_masks, val_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDO34zRNfYZy"
      },
      "source": [
        "# Save the tensorDatasets\n",
        "torch.save(train_dataset, '/content/drive/MyDrive/Yelp/model_128_/_train_128')\n",
        "torch.save(val_dataset, '/content/drive/MyDrive/Yelp/model_128_/_test_128')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bh_SUlsHaDs1"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch \n",
        "# size of 16 or 32.\n",
        "batch_size = 16\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order. \n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially. FROM THE 2ND DIVISION\n",
        "            batch_size = batch_size # Evaluate with this batch size.\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0NQLbIAKTZ4y"
      },
      "source": [
        "### Intializing BertForSequenceClassification model and helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RASREEhaaTrr",
        "outputId": "3569fcac-78a4-4612-e357-b5f16cdcc8c9"
      },
      "source": [
        "%%time\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification. # It can be increased for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3.29 s, sys: 197 ms, total: 3.48 s\n",
            "Wall time: 3.66 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "slORUUcnayjI"
      },
      "source": [
        "Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DL_Efz0IaqbZ"
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WOVjafvPYRAq"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4. \n",
        "epochs = 3\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs]. \n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xwg-r_jBqKbH"
      },
      "source": [
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XiJn1ohNYri1"
      },
      "source": [
        "import datetime\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQztPN7Ga9LS"
      },
      "source": [
        "import random\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "44x8AkQrcQaa"
      },
      "source": [
        "# Training\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2nKomsf_JMPj",
        "outputId": "a0df9bdb-10d8-4eff-9c53-8015efa8194d"
      },
      "source": [
        "# We'll store a number of quantities such as training and validation loss, \n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "         \n",
        "\n",
        "      # Perform a forward pass (evaluate the model on this training batch).\n",
        "      # The documentation for this `model` function is here: \n",
        "      # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "      # It returns different numbers of parameters depending on what arguments\n",
        "      # arge given and what flags are set. For our useage here, it returns\n",
        "      # the loss (because we provided labels) and the \"logits\"--the model\n",
        "      # outputs prior to activation.\n",
        "        loss_logits_ = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask, \n",
        "                            labels=b_labels)\n",
        "\n",
        "        loss = loss_logits_[0]\n",
        "        logits_t = loss_logits_[1]\n",
        "\n",
        "        logits_t_cpu = logits_t.detach().cpu().numpy()\n",
        "        train_pred = np.argmax(logits_t_cpu, axis=1).flatten()\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here: \n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "            # values prior to applying an activation function like the softmax.\n",
        "            loss_logits = model(b_input_ids, \n",
        "                                   token_type_ids=None, \n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels)\n",
        "            loss = loss_logits[0]\n",
        "            logits_val = loss_logits[1]\n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits_val = logits_val.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        val_pred = np.argmax(logits_val, axis=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits_val, label_ids)\n",
        "        \n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    \n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of    650.    Elapsed: 0:00:09.\n",
            "  Batch    80  of    650.    Elapsed: 0:00:17.\n",
            "  Batch   120  of    650.    Elapsed: 0:00:26.\n",
            "  Batch   160  of    650.    Elapsed: 0:00:35.\n",
            "  Batch   200  of    650.    Elapsed: 0:00:44.\n",
            "  Batch   240  of    650.    Elapsed: 0:00:52.\n",
            "  Batch   280  of    650.    Elapsed: 0:01:01.\n",
            "  Batch   320  of    650.    Elapsed: 0:01:10.\n",
            "  Batch   360  of    650.    Elapsed: 0:01:18.\n",
            "  Batch   400  of    650.    Elapsed: 0:01:27.\n",
            "  Batch   440  of    650.    Elapsed: 0:01:36.\n",
            "  Batch   480  of    650.    Elapsed: 0:01:45.\n",
            "  Batch   520  of    650.    Elapsed: 0:01:53.\n",
            "  Batch   560  of    650.    Elapsed: 0:02:02.\n",
            "  Batch   600  of    650.    Elapsed: 0:02:11.\n",
            "  Batch   640  of    650.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.29\n",
            "  Training epcoh took: 0:02:22\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.90\n",
            "  Validation Loss: 0.27\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of    650.    Elapsed: 0:00:09.\n",
            "  Batch    80  of    650.    Elapsed: 0:00:17.\n",
            "  Batch   120  of    650.    Elapsed: 0:00:26.\n",
            "  Batch   160  of    650.    Elapsed: 0:00:35.\n",
            "  Batch   200  of    650.    Elapsed: 0:00:43.\n",
            "  Batch   240  of    650.    Elapsed: 0:00:52.\n",
            "  Batch   280  of    650.    Elapsed: 0:01:01.\n",
            "  Batch   320  of    650.    Elapsed: 0:01:10.\n",
            "  Batch   360  of    650.    Elapsed: 0:01:18.\n",
            "  Batch   400  of    650.    Elapsed: 0:01:27.\n",
            "  Batch   440  of    650.    Elapsed: 0:01:36.\n",
            "  Batch   480  of    650.    Elapsed: 0:01:44.\n",
            "  Batch   520  of    650.    Elapsed: 0:01:53.\n",
            "  Batch   560  of    650.    Elapsed: 0:02:02.\n",
            "  Batch   600  of    650.    Elapsed: 0:02:10.\n",
            "  Batch   640  of    650.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.16\n",
            "  Training epcoh took: 0:02:21\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.92\n",
            "  Validation Loss: 0.26\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of    650.    Elapsed: 0:00:09.\n",
            "  Batch    80  of    650.    Elapsed: 0:00:17.\n",
            "  Batch   120  of    650.    Elapsed: 0:00:26.\n",
            "  Batch   160  of    650.    Elapsed: 0:00:35.\n",
            "  Batch   200  of    650.    Elapsed: 0:00:43.\n",
            "  Batch   240  of    650.    Elapsed: 0:00:52.\n",
            "  Batch   280  of    650.    Elapsed: 0:01:01.\n",
            "  Batch   320  of    650.    Elapsed: 0:01:09.\n",
            "  Batch   360  of    650.    Elapsed: 0:01:18.\n",
            "  Batch   400  of    650.    Elapsed: 0:01:27.\n",
            "  Batch   440  of    650.    Elapsed: 0:01:35.\n",
            "  Batch   480  of    650.    Elapsed: 0:01:44.\n",
            "  Batch   520  of    650.    Elapsed: 0:01:52.\n",
            "  Batch   560  of    650.    Elapsed: 0:02:01.\n",
            "  Batch   600  of    650.    Elapsed: 0:02:10.\n",
            "  Batch   640  of    650.    Elapsed: 0:02:18.\n",
            "\n",
            "  Average training loss: 0.09\n",
            "  Training epcoh took: 0:02:20\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.92\n",
            "  Validation Loss: 0.34\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:07:34 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z3GSlwbCsC46"
      },
      "source": [
        "### Saving model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adlJFThIK-O9"
      },
      "source": [
        "tokenizer.save_pretrained('/content/drive/MyDrive/Yelp/model_128_/tokenizer_128')\n",
        "torch.save(model, '/content/drive/MyDrive/Yelp/model_128_/tokenizer_128/model-02-FEB_3e')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "obswJqqfJdxp"
      },
      "source": [
        "# Summary of training process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "pj6G5e3EPlwu",
        "outputId": "0c128003-c676-4823-e3ba-4e78863e9150"
      },
      "source": [
        "# stats of max length = 128 and 3 epochs and batch size = 16\n",
        "# Display floats with two decimal places.\n",
        "pd.set_option('precision', 2)\n",
        "\n",
        "# Create a DataFrame from our training statistics.\n",
        "df_stats_128 = pd.DataFrame(data=training_stats)\n",
        "\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats_128 = df_stats_128.set_index('epoch')\n",
        "\n",
        "# A hack to force the column headers to wrap.\n",
        "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
        "\n",
        "# Display the table.\n",
        "df_stats_128"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.29</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0:02:22</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.16</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0:02:21</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.09</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0:02:20</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Valid. Loss  Valid. Accur. Training Time Validation Time\n",
              "epoch                                                                         \n",
              "1               0.29         0.27           0.90       0:02:22         0:00:10\n",
              "2               0.16         0.26           0.92       0:02:21         0:00:10\n",
              "3               0.09         0.34           0.92       0:02:20         0:00:10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kjU2LKj4vCmQ",
        "outputId": "e3bcabb5-8056-46b7-8513-2c6a59b4bd4a"
      },
      "source": [
        "df_stats_128[['Training Loss', 'Valid. Loss', 'Valid. Accur.']].mean().apply(lambda x: round(x, 2)) #new"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Training Loss    0.18\n",
              "Valid. Loss      0.29\n",
              "Valid. Accur.    0.91\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htWT_EWYpLe2"
      },
      "source": [
        "df_stats_128.to_pickle('/content/drive/MyDrive/Yelp/loss_accuracy_results_2df/stats_128_3e_16')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "s32CxN4dw9xy",
        "outputId": "f5ff44a0-1fa2-4184-96b6-17a3d11066cc"
      },
      "source": [
        "# reults of max length = 128 and 3 epochs and batch size = 16\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats_128['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats_128['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss | max_len=128, epochs=3, batch_size=16\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3])\n",
        "plt.savefig('/content/drive/MyDrive/Yelp/loss_accuracy_results_2df/loss_128_3e_16', format='jpg') #saving the graph\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxM1/vA8U8meyIRIrEk1lQSsoldpfYQS0qJUmvRorW0fFtbN/y+ulhKLdWiVRRBlorWHlS1lqogiC0UkUhCZCXr3N8f6czXmCAhMaHP+/XyanPm3HvPvXPnzjP3PuccI0VRFIQQQgghhBDPBJWhGyCEEEIIIYQoPgnghRBCCCGEeIZIAC+EEEIIIcQzRAJ4IYQQQgghniESwAshhBBCCPEMkQBeCCGEEEKIZ4gE8P9ycXFxuLm5sWjRosdex5QpU3BzcyvFVj2/HnS83dzcmDJlSrHWsWjRItzc3IiLiyv19oWFheHm5sbhw4dLfd3PO8378m/RoUMHBg8ebOhmiHKgNL5HnhXl/RppqM+lXA+ePhNDN0DoKkkAEBkZibOzcxm25tlz584dvvnmG7Zu3UpSUhKVK1emSZMmvP3227i4uBRrHePHj2fHjh389NNPNGjQoMg6iqLQsWNH0tPTOXDgABYWFqW5G2Xq8OHDHDlyhKFDh2Jra2vo5uiJi4ujY8eODBw4kI8//tjQzRHPgW+//ZbTp09z+vRp4uLicHJyYs+ePXr1cnJy2Lx5M3v37uXcuXPcvHkTBwcHfHx8GDNmTJHXkJSUFFasWMGePXtISEigQoUKuLi4MGTIEDp16vQ0dk/c59KlSyxZsoQzZ86QlJREfn4+1atXp23btowYMQJHR0dDN7FIYWFhpKen8/rrrxu6Kc+l4l4H7vXTTz8RHBzM+fPnURQFJycnunbtypgxY55Sqx9MAvhyZvbs2Tp///XXX2zYsIF+/frRpEkTndcqV678xNtzcnLi5MmTGBsbP/Y6/u///o8ZM2Y8cVtKw4cffsgvv/xCjx49aN68OcnJyezZs4cTJ04UO4APCgpix44dhIaG8uGHHxZZ59ChQ1y/fp1+/fqVSvB+8uRJVKqn80DsyJEjLF68mFdeeUUvgO/Zsyfdu3fH1NT0qbRFiKfhyy+/xM7OjoYNG5KRkfHAenFxcXz00Uc0adKEPn364OjoSFxcHOvXr2fnzp2sWLGCli1bauvfvXuX1157jYSEBF599VXc3NxITU0lPDycMWPG8MknnzBgwICnsYviHomJiSQnJ+Pv70/VqlUxMTHh/PnzbNy4kV9++YXNmzdjb29v6GbqCQ8P5/r1689kAL99+3ZDN+GRinsd0Jg6dSo//fQTnTt35uWXX0alUhEXF0d8fPxTaO2jSQBfzvTs2VPn74KCAjZs2ECjRo30XrtfZmYmFSpUKNH2jIyMMDc3L3E771Vegr27d++yfft2/Pz8mDdvnrZ87Nix5ObmFns9fn5+VK9enS1btjBp0iTMzMz06oSFhQGFwX5peNL3oLQYGxs/0Y85Icqj3bt3U7NmTQB69OjBnTt3iqxXuXLlIp+8BQYG0qtXL2bPnq397EPhU9C///6badOmMXToUG15v379aNu2LRs2bJAA3gBatWpFq1at9MqbNm3Ku+++S1hYGG+++aYBWvb8Kup7srwp7nUAYNOmTYSFhfHFF1/Qq1evp9XEEpEc+GeUJt/szJkzjBgxgiZNmvDyyy8DhYH8/Pnz6du3Ly1atMDT0xN/f3/mzp3L3bt3ddZTVO7ivWV79+6lT58+eHl54efnxxdffEF+fr7OOorKgdeUZWRk8Mknn9CqVSu8vLzo378/J06c0Nuf27dvM3XqVFq0aIGvry9DhgzhzJkzDB48mA4dOhTrmBgZGWFkZFTkD4qSXFxUKhWvvPIKqampRT5ey8zMZOfOnbi6uuLt7V2i4/0gReXAq9Vqvv32Wzp06ICXlxc9evQgIiKiyOVjY2OZPn063bt3x9fXFx8fH3r37s2mTZt06k2ZMoXFixcD0LFjR9zc3HTe/wfld6akpDBjxgzatm2Lp6cnbdu2ZcaMGdy+fVunnmb5gwcP8t1339GpUyc8PT3p0qUL4eHhxToWJXH27FnGjBlDixYt8PLyolu3bixfvpyCggKdegkJCUydOpX27dvj6elJq1at6N+/v06b1Go1P/zwA4GBgfj6+tK4cWO6dOnCtGnTyMvLK9V233ucFi9eTPv27fH29qZv374cP34cKHxS8tprr9GoUSP8/PxYsmSJ3noOHDjAu+++S8eOHfH29qZp06YMHz6cI0eO6NQ7c+YMXl5eDBs2DEVRtOUFBQUMHDgQHx8fzp8/Xyr7Fh0drX1PNO/90qVL9a4bms92YmIiEydOpFmzZvj4+DBixAguX75cKm3R0HxpP0qlSpWKTJt74YUXcHV15cKFCzrlmZmZAHopGTY2NlhaWmJpafmYLS6UlJTEJ598Qrt27fD09MTPz4+PPvqIW7du6dTT9L+4cOEC//3vf2ndurX2fDp48GCR6960aROvvPIK3t7eNGnShOHDh3P06NEi6x46dIiRI0dqP2cdO3Zk2rRppKSk6NUtznfGhQsXGD9+PC+99BKenp60bt2awYMHs2/fvsc7UMXk5OQEQHp6+hOvq6CggEWLFmmvKYGBgfzyyy969Yr7Ge3QoQNHjhzh+vXr2uvy/dfiK1euMHXqVNq0aaM9H9566y1OnTqlt93Y2FhGjhyJr68vTZo0Yfz48SQnJ5d4P3Nycli0aBFdunTBx8eHpk2bEhgYyBdffKHX/ntz4DXn5IP+3duHKyMjgzlz5uDv74+npyctW7Zk4sSJXLt2rcTtfZjiXgcURWHZsmV4eHhog/fMzEyda2d5IHfgn2Hx8fEMHTqUgIAAOnfurP01mZiYSEhICJ07d6ZHjx6YmJhw5MgRVqxYQUxMDN99912x1v/rr7+ybt06+vfvT58+fYiMjOT777+nYsWKjB49uljrGDFiBJUrV2bMmDGkpqaycuVKRo4cSWRkpPZpQW5uLsOGDSMmJobevXvj5eXFuXPnGDZsGBUrViz28bCwsKBXr16Ehoby888/06NHj2Ive7/evXuzdOlSwsLCCAgI0Hntl19+ITs7mz59+gCld7zv99lnn7F69WqaNWvG66+/zq1bt5g5c2aRF6EjR45w9OhR2rVrh7Ozs/ZpxIcffkhKSgqjRo0CCu8MZmZmsmvXLqZOnUqlSpWAh/e9yMjI4LXXXuPKlSv06dOHhg0bEhMTw/r16zl06BCbNm3Se/Izf/58srOz6devH2ZmZqxfv54pU6ZQq1YtvVSwxxUdHc3gwYMxMTFh4MCBVKlShb179zJ37lzOnj2rfQqTn5/PsGHDSExMZMCAAdSpU4fMzEzOnTvH0aNHeeWVVwBYunQpCxcupH379vTv3x9jY2Pi4uLYs2cPubm5ZfKkae7cuajVaoYMGUJeXh7ff/89w4cPZ/bs2XzwwQe8+uqrBAYGsm3bNhYuXIizs7POk7jw8HDS0tLo1asX1apVIzExkU2bNvH666+zevVqmjZtCkDDhg15//33mTVrFsuWLdOeD4sXL+bo0aPMmDEDV1fXJ96fffv2MXbsWGrXrs3w4cOpWLEix48fZ+HChcTExLBw4UKd+nfu3GHQoEH4+PgwYcIE4uLiWL16NW+//TY///yz9mmQWq0mNTW12O2ws7Mr1ZQ0tVpNUlISVapU0Slv2bIlJiYmfPnll1hZWeHm5kZaWho//PAD6enpvPXWW4+9zfj4ePr160deXh5BQUHUqlWLK1eusH79eg4fPkxoaCg2NjY6y0yePBmVSsWbb75JZmYmGzZs4I033mD58uW8+OKL2npz5sxhxYoVeHt7M3HiRDIzM9m4cSNDhw7l66+/pm3bttq6wcHBTJ8+napVq9K/f3+cnJyIj49n7969JCYm6qRyFuc74/bt29qnFf3796dGjRrcvn2bU6dOceLECdq1a6c95k/6nufk5JCVlUVubi4XL15k7ty5ADr797jmzp3LnTt3eO2114DCH+UTJ04kJyeH3r17a+sV9zM6bdo05s2bp72ZpaFJ/YyOjub1118nPz+foKAg6tevT1paGkeOHCEqKgpPT0/tMomJido+GJMmTeLs2bNs2LCBzMxMvv/++xLt54wZMwgNDaVXr174+vpSUFDA33///chOvP7+/tSqVUunLDc3l88//5yCggKsra2Bwu+X/v37Ex8fT58+fahfvz7JycmsW7eOvn37Ehoaqv3h9bSuA5cuXeLq1asMGjSIJUuWsHr1alJTU6lQoQLdu3dn8uTJ2vYblCLKtdDQUMXV1VUJDQ3VKW/fvr3i6uqqbNy4UW+ZnJwcJTc3V698/vz5iqurq3LixAlt2bVr1xRXV1dl4cKFemU+Pj7KtWvXtOVqtVrp3r270rp1a531Tp48WXF1dS2y7JNPPtEp37p1q+Lq6qqsX79eW/bjjz8qrq6uytdff61TV1Pevn17vX0pSkZGhvLmm28qnp6eSsOGDZVffvmlWMs9yJAhQ5QGDRooiYmJOuWvvvqq4uHhody6dUtRlCc/3oqiKK6ursrkyZO1f8fGxipubm7KkCFDlPz8fG35qVOnFDc3N8XV1VXnvcnKytLbfkFBgTJo0CClcePGOu1buHCh3vIamvPt0KFD2rIvv/xScXV1VX788Uedupr3Z/78+XrL9+zZU8nJydGW37hxQ/Hw8FAmTJigt837aY7RjBkzHlqvX79+SoMGDZSYmBhtmVqtVsaPH6+4uroqf/zxh6IoihITE6O4uroqy5Yte+j6evXqpXTt2vWR7XsQzXEtDs1x6tWrl85x2r17t+Lq6qo0bNhQOXnypLY8JydHad26tfLqq6/qrKeo9z05OVlp3ry58sYbb+i9Nnr0aKVhw4ZKVFSUcujQIcXd3V0ZN25ccXdRR/v27ZVBgwZp/87OzlZefPFFZcCAAUpeXp5O3ZUrV+qdV4MGDSryfVm+fLni6uqq7N+/X1umOSeK+6+oc1uje/fuxb6maGjO9QULFui9tmPHDqVNmzY623/xxReVo0ePlmgb9xs9erTSsmVLJSEhQaf85MmTSoMGDXSuIZpzLygoSOd8SkhIUBo1aqQEBARoyzTXlv79++t9Rps0aaK0b99ee81JSEhQPDw8lK5duyppaWl6bSwoKFAUpWTfGZpz/FHX59J4z9esWaNTp3379srmzZsfut1H0Xx227Vrp6Snp2vL09PTlXbt2inNmjVT7t69qy0vyWd00KBBRZ6bmuPo6empc73T0LwPivK/2OD+4zt9+nTF1dVViY2NLf7OKorSrFmzIq8l97v/enA/tVqtvPvuu4qbm5uyc+dObfn//d//KV5eXnr7FRcXp/j6+up8Lz6t68CuXbsUV1dXpWXLlkqTJk2UFStWKNu3b1emTJmiuLq6KoMGDVLUavUjj0lZkzvwzzA7OzudX/oa96aL5Ofnk5WVRUFBAS+++CJLly7lxIkTeHt7P3L9HTt21BnlxsjIiBYtWvDjjz+SlZVVrF+g93fG0XQAu3LlirZs7969GBsbM2TIEJ26ffv2Zf78+Y/cBhT+Mn/nnXe4cOEC27ZtY9asWbz33nsUFBQQGBiorffRRx8RFhZGVFTUI9NqgoKCOHToED/99BMjR44ECh9LHj9+nC5dumjvPJXW8b5XZGQkiqIwbNgwnZx0Dw8PWrduzYEDB3TqW1lZaf8/JyeHO3fuoCgKrVu35siRI1y6dOmxhzjctWsXlStXpl+/fjrl/fr1Y/HixezevZt3331X57UBAwboHJeqVatSt25d/v7778dqw/1u3bpFVFQU/v7+uLu7a8uNjIx466232L59O7t27aJVq1bau5SHDx+md+/eD+y8VqFCBa5evcrRo0e1d8XK2muvvaZznDTb9fb2xsvLS1tuZmaGl5cXx44d01n+3vddc6dRpVLh4+NTZKraZ599Rs+ePZk4caJ2ZI7//ve/pbIvv//+Ozdv3mTixIl6KQpt2rThs88+4/fff6dFixbacpVKpfe5v/ca8dJLLwHg4ODAypUri90WBweHx90NPceOHePzzz/H3d29yCePNjY2uLm50bdvX9zd3UlMTGTlypW8/fbbrFq1Suf8LK6MjAz27dtH7969MTMz00lVcXJyolatWvz++++MGzdOZ7nXX39d53yqVq0agYGBbNiwgdjYWFxcXLTXljfeeEPvM9q7d29WrVqlTbnavn07eXl5jB07tsgRq+6/u1mc7wzN5/G3336jTZs2D+y3VRrveadOnahXrx537tzhzJkz7NmzRy/t73G99tprOk9AbGxs6N+/P19++SWHDx/W3uUv6We0KDExMVy4cIH+/fsXeT7d/z44OjrSrVs3nbKWLVuybt06rly5Qr169Yq9nxUqVODixYucP3/+iZ7SLViwgK1bt/Lee+/h7+8PFKaqbNmyhWbNmuHo6KhznltaWtKoUSOd77qndR3IysoCClNHV65cqX161aVLFxRFITw8nP3795fKk5wnIQH8M6xmzZoP7HC4du1agoODuXjxImq1Wue1tLS0Yq//fnZ2dgCkpqYWK4C/fx2alI17H4PFxcXh6Oiotz4zMzOcnZ2Lla8YGRnJgQMHmDNnDs7Oznz11VeMHTuWSZMmkZ+fr02TOHfuHF5eXsXKie/cuTO2traEhYVpA/jQ0FAAbfqMRmkc73tpcv+KutC6uLjoBfBZWVksXryYbdu2kZCQoLfMk+R8xsXF4enpiYmJ7uXCxMSEOnXqcObMGb1lHnTuXL9+/bHbcX+boDA3+X716tVDpVJpj6GTkxOjR49m2bJl+Pn50aBBA1q2bElAQIDOD6uJEycyZswYBg4ciKOjI82bN6ddu3Z06dKlzDpo3X+cNCljRQ0PW7FiRb3Hx1evXmX+/PkcOHBA7z02MjLSW4ednR2fffYZw4YNA2DdunWlNpRobGwsUJgK8CA3b97U+dvR0VGvA/e91xgNc3NznRSQp+XUqVOMGjUKR0dHli1bptfW3377jZEjR/Ltt9/Spk0bbXnnzp3p2rUrM2bMYP369SXe7uXLl1Gr1YSEhBASElJknaI+Y0WNtKUpu3btGi4uLtrPTv369fXqasquXbuGl5eX9gf3g4bTLU6b7v/OaN68Ob169SIsLIwtW7bg6enJiy++SLdu3XQ+z6XxnlerVo1q1aoBhcF8586dCQoKIjs7W5tG9rgedG0GdPK7S/oZLYrmfWjYsGGx6j/qfSiJadOmMWnSJAIDA6lZsyYtWrSgffv2dOjQodjpKeHh4XzzzTcEBQXpdB5OSUkhNTWVAwcOFNnpGHR/nDyt64BmZLmqVavqba9Xr16Eh4dz5MgRCeDF43tQB6mVK1fy+eef4+fnx5AhQ3B0dMTU1JTExESmTJlS7I4YDxuN5EnXUdzli0uTj9esWTOgMPhfvHgxb731FlOnTiU/Px93d3dOnDjBrFmzirVOc3NzevTowbp16zh27Bg+Pj5ERERQrVo17Z1BKL3j/ST+85//sG/fPl599VWaNWuGnZ0dxsbG/Prrr/zwww96PyrK2tMaErO4JkyYQFBQEPv27ePo0aOEhITw3Xff8cYbb/D+++8D4Ovry65duzhw4ACHDx/m8OHD/PzzzyxdupR169ZpvwBL04OOU3FGAsrKymLgwIHcvXuXoUOH4urqirW1NSqVim+//ZZDhw4Vudy9HbPPnj1ban0SNOf5pEmTHhjw3d/Zs7jXmIKCgiI7TD5I5cqVn3g0pdOnTzN8+HBsbGxYvXo1VatW1auzfPlyLC0tdYJ3KLzz17RpU/bv309ubm6JfwBq9v3ll1/W3ny4X3kZuepexX0/v/jiC0aMGMH+/fs5evQoK1eu5JtvvmHatGkMGjQIKJv33N3dnYYNG7Ju3bonDuCL43E/o0+qNL67NTp16sSePXv49ddf+fPPP/njjz8ICQmhadOmrFy58pHn9uHDh/noo49o2bIl06dPL7ItL774YrFGBXpa1wHNj777+7zA/+7ql0ZH6CclAfxzaPPmzTg5ObF8+XKdAGH//v0GbNWDOTk5cfDgQb20nLy8POLi4op1h1Czn9evX6d69epAYRD/9ddfM3r0aD766COcnJxwdXUt0ZBQQUFBrFu3jrCwMNLS0khOTmb06NE6x7UsjrfmDsqlS5f0OgJp7nRqpKens2/fPnr27MnMmTN1Xvvjjz/01l3cuz73tuXy5cvk5+fr3IXPz8/n77//LnbP/tKkuUN98eJFvdcuXbqEWq3Wa1fNmjUZPHgwgwcPJicnhxEjRrBixQqGDx+uTauxtramS5cudOnSBSh8sjJz5kxCQkJ44403ynivSubgwYMkJSXx6aef6j0RWrBgQZHLREZGsmbNGnr37s3Vq1f54osvaNq0aanMIFunTh2g8MZCad8lS0hIoGPHjsWu/6ST3J0+fZphw4ZhbW3NqlWrtJ3o7peYmIiiKCiKove5ys/Pp6Cg4LF+wNeqVQsjIyPy8vJKdCxjY2P1Uiw01wvN50Hz3wsXLuhdWzSfJ00dzXsaExND3bp1S7wfD+Pq6oqrqytvvPEG6enp9O3bl3nz5jFw4ECMjIzK7D3Pzs5+rKei97t06ZJemeZYa9rxOJ/RomiOfUxMzOM294nY2dnRs2dPevbsiaIozJ07lxUrVhAZGUnXrl0fuNylS5cYN24czs7OLFy4UG8ggMqVK2Nra0tmZmaxzvOndR1wdXXF3NycpKQkvdcSExOB0pmH50lJAP8cUqlUGBkZ6Xxx5Ofns3z5cgO26sE6dOjAgQMHWL16tc6oDRs3biQjI6NYAXzbtm1ZtWoV8+fP17krYG5uzpdffkmXLl2Ii4tj3LhxeqkgD+Ph4UGDBg3YunUrN27cwMjISG/s97I43h06dGDevHmsXLmSl156SXsX4fTp03pBueZHw/2BQlJSkt4wkvC/nMy0tLRiXdw6derEN998w6ZNm7QjLkDh+5OSkqKXG/802Nvb4+vry969e3VyM5V/hv8CtHmWGRkZWFhY6Hx5mJubU69ePf7880/S0tKwt7cnJSVF76Ls4eEBPF4aVFnTnBP3v+8HDhwoMrc2MTGRadOm4eLiwscff0xqaqo2Hz40NPSJJyTz8/PD3t6e5cuX061bN70nFtnZ2eTn55d4rgp4ujnwZ86cYfjw4VhZWbF69eqH/kB94YUX+Pvvv9m2bZtOzvG1a9c4evSoNhAoqUqVKtG2bVt27drF8ePHadSokc7riqJw+/ZtvfP1hx9+wN/fX3v9u3HjBlu2bKFu3bra9I4OHTowd+5cvvvuO9q0aaP9XCQlJREWFoaTk5M2VSMgIIC5c+eyZMmSIvPVi/rh8iipqanY2trq3OywtbXF2dmZK1eukJOTg4WFxRO958nJyUWeA4cOHeLChQs0b968RG0uyvr163Xy4DMyMggODsbW1la7/pJ+Rq2trUlLS9M7ru7u7tSvX5/Q0FAGDhyol/70OO9DcRQUFJCVlaXzHWxkZKQ9Px52Xbx9+zajRo3CyMiIZcuWFTminEqlIjAwkLVr17J9+3a9Ed+gsL+T5gbL07oOWFpa0rlzZ7Zs2cKuXbu03yWANiXO0OkzIAH8cykgIIB58+bx5ptv4u/vT2ZmJj///HOJAtenqW/fvgQHB7NgwQKuXr2qHUZy+/bt1K5dW28M4aK0bt2aoKAgQkJC6N69Oz179qRatWpcu3aNzZs3A4XB2JIlS3BxcXnoXYP7BQUF8X//93/89ttvNG/eXO8LvSyOt4uLCwMHDuTHH39k6NChdO7cmVu3brF27Vrc3d118s4rVKhA69atiYiIwMLCAi8vL65fv86GDRtwdnbWy3n08fEBCodBCwwMxNzcnPr16z+wg9Ibb7zB9u3bmTlzJmfOnKFBgwbExMQQEhJC3bp1y+zO9KlTp/j666/1yk1MTBg5ciQffPABgwcPZuDAgQwYMAAHBwf27t3LgQMH6NGjhzanUvMIt3PnztStWxdra2tOnTpFSEgIPj4+2lzWbt260ahRI7y9vXF0dCQ5OZmNGzdiampK9+7dy2Qfn0STJk1wcHDgiy++4Pr161SrVo2YmBg2b96Mq6urzrjuarWa9957j7t37zJ//nztGOWzZs1i7NixfPrpp3pPb0rKysqKL774gjFjxhAQEECfPn2oXbs26enpXLp0iV27drF48WKdTqzF9aS5rz/99JN29sSUlBTy8vK051aNGjW0T+WuX7/OsGHDSEtLY/DgwRw7dkyv47C/v7/2R/CoUaP47bffmDRpEkeOHKFBgwbcuHGD9evXk5OTw8SJE3WWnTJlCuHh4axevfqRx2H69OkMGDCAQYMG0bNnTxo2bIharebatWtERkbSq1cvvU6smnH9u3fvTlZWFsHBweTk5OjMKF2vXj3t06dBgwbRtWtXsrKy2LhxI3fu3GHu3LnawLNatWpMmzaNmTNnEhgYSM+ePXFyciIxMZHIyEg+/fTTYufH3/terFq1ik6dOlG7dm1MTEz4888/OXDgAF27dtX+kHyS93z69OkkJyfTsmVLatSoQU5ODqdPn2br1q1YW1vrzbkRFhbG1KlTGTt2rN4xfZBKlSrRt29f7UASYWFhxMfH89///leb3lqSzygUXpv37t3LzJkz8fX1xdjYmJYtW2Jvb8+nn37K66+/Tt++fbXDSKanp/Pnn3/y0ksv6YzBXlqysrLw8/OjQ4cONGzYkMqVK2tnJq5YsSLt27d/4LIzZszg6tWr9O/fn6ioKKKionRe13yOJkyYwLFjx3j33Xfp2rUrPj4+mJqaEh8fz/79+/Hw8ODzzz8Hnt51AAr7RB08eJD//Oc/DBo0CCcnJ/bv38++ffvo1asXjRs3fux2lJbyGdGJJzJixAgURSEkJIRZs2bh4OBA165d6dOnj17P9PLAzMyMVatWMXv2bCIjI9m2bRve3t788MMPfPDBB2RnZxdrPbNmzaJ58+YEBwfz3XffkZeXh5OTEwEBAQwfPhwzMzP69evH+++/j42NDX5+fsVab2BgIJhUE8gAACAASURBVLNnzyYnJ0fvMSiU3fH+4IMPqFKlChs3bmT27NnUqVOHjz/+mCtXruh1HJ0zZw7z5s1jz549hIeHU6dOHSZMmICJiYnOmMJQ+KXy3nvvERwczEcffUR+fj5jx459YABvY2PD+vXrWbhwIXv27CEsLAx7e3v69+/PuHHjHuuOanGcOHGiyLtUZmZmjBw5Ei8vL4KDg1m4cCHr16/nzp071KxZk/fee4/hw4dr67u5ueHv78+RI0fYsmULarWa6tWrM2rUKJ16w4cP59dff2XNmjVkZGRgb2+Pj48Po0aNeqyRRMqara0tK1asYM6cOfz444/k5+fj6enJ8uXLCQkJ0QkOvv76a44cOcInn3yiky7j7+/PgAEDWLduHa1bt9amDj2ul156iZCQEJYtW0ZERAS3b9/G1taWWrVq8frrr5dKqs7jCA0N1Zs456uvvgLQdqqEws6Hmh+8905ud6/IyEhtAO/t7U1wcDDffPMNO3fuZOPGjVhbW+Pt7a2d+OheWVlZGBkZFevOYPXq1QkNDWX58uXs2bOHiIgIzM3NqV69Ou3bty/yJsQXX3xBcHAwy5cvJz09HTc3Nz7//HNat26tU+/999+ndu3arFu3jnnz5mFqaoqPjw/z5s3TG4FpwIAB1KpVi++++441a9aQm5uLo6MjrVq10uYKl0SLFi2IiYlh3759JCcno1KpcHZ2ZvLkydr89yfVvXt3Nm/ezObNm0lJScHIyIgaNWrQr18/RowYQY0aNXTqa0YdKaqfw4O89957HD16lHXr1nHz5k3q1q2rvSmiUZLPKBSOInTt2jV27NhBcHAwarWa1atXY29vj7e3NyEhIXz99dds27aN4OBg7Ozs8Pb2LrNg0sLCgqFDh3Lw4EFtmqujoyMdOnRg1KhRDz1emsnGgoODCQ4O1ntd8znSfL98//33bN++ncjISIyNjalWrRpNmjShb9++pbY/xb0OQGFAv2HDBubPn09YWBiZmZnUrFmTyZMn642uZyhGytPoYSfEYygoKKBly5Z4e3s/9mRIQjwtixYtYvHixZw7d87QTRHlkFqtplWrVrRr105vFssnpTn3njT3/99q7NixnD9/nl9++aVMJmwToiyUr6EixL9WUXfZg4ODSU9P17t7JIQQz5pTp06RnZ3NhAkTDN0UcY/8/HwOHTrEf/7zHwnexTNFUmhEufDhhx+Sm5uLr68vZmZmREVF8fPPP1O7dm1effVVQzdPiH+FlJQUCgoKHlrHysqqfEwj/ozx9vYu9sQ94ukxMTHh6NGjhm7GU5Wbm1usjvmlMRyrKDsGDeBzc3P56quv2Lx5M+np6bi7uzNhwoQHDuivERERQUhICLGxsaSlpeHo6EiLFi0YO3as3nBfD8q5nD59us6IGsKw/Pz8WLt2LQcPHuTOnTvY29vTt29f3nnnnTLLsRZC6AoKCnrkZFsl6egnhCh/oqKi9GZALoqkZJVvBs2BnzhxIjt37mTIkCHUrl2b8PBwTp06xZo1a/D19X3gcrNnzyY5ORl3d3cqVqxIfHw8GzdupKCggIiICJ0OQm5ubvj5+fHyyy/rrMPHx0c7xq0QQgj466+/yMnJeWidmjVrGmTsfyFE6UhLS+P06dOPrNekSZNyOWGYKGSwAP7kyZP07duXqVOnanv05uTk0KNHDxwdHVm7dm2J1nf69Gl69+7NpEmTGDFihLbczc2NIUOG8MEHH5Rm84UQQgghhDAIg3Vi3b59O6ampjpDBJmbmxMUFMRff/1V5AxYD6MZFupB09tmZ2c/8s6SEEIIIYQQ5Z3BcuA1UzPf3xnK29sbRVGIiYnB0dHxoetITU2loKCA+Ph4lixZAlBk/nxISAhr1qxBURRcXV0ZP368zsxaJXH7dhZq9dN/aGFvX4FbtzKf+naFEOJJyfVLCPEsM8Q1TKUyolKlBw8YYLAAPjk5uchJADT568W5A9+lSxftpBt2dnZ8/PHHtGzZUqeOr68v3bp1w9nZmYSEBFavXs3YsWOZN28ePXr0KHG71WrFIAG8ZttCCPEskuuXEOJZVt6uYQYL4LOzs4scc1XTYaI46S6LFy/mzp07XL58mYiICO1save6fwawV155hR49ejBnzhy6d++OkZFRidptb2+4EVEcHGwMtm0hhHgScv0SQjzLyts1zGABvIWFBXl5eXrlmsC9OD2fmzVrBkDbtm3p2LEjgYGBWFlZPXQ6ZisrK/r378+8efO4dOkSLi4uJWr3rVuZBvkV5uBgQ3JyxlPfrhBCPCm5fgkhnmWGuIapVEYPvWlssE6sDg4ORabJJCcnAzwy//1+NWvWxMPDgy1btjyybvXq1QGKNZGBEEIIIYQQ5YnBAnh3d3cuX76sl/aimanO3d29xOvMzs4mI+PRv5CuXbsGFM4yJoQQQgghxLPEYAF8QEAAeXl5bNq0SVuWm5tLWFgYjRs31nZwjY+PJzY2VmfZlJQUvfWdOnWKs2fP4uHh8dB6t2/fZt26dTg7O8tETkIIIYQQ4pljsBx4Hx8fAgICmDt3LsnJydSqVYvw8HDi4+P57LPPtPUmT57MkSNHOHfunLasffv2dO3aFVdXV6ysrLh48SKhoaFYW1vz9ttva+utXbuWyMhI2rVrR40aNUhMTGTDhg2kpKRoh50sC3fvZpGZmUZBgX6O/+NKSlKhVqtLbX3CsIyNTalQoSKWlg8eIkoIIYQQoigGC+ABZs+ezYIFC9i8eTNpaWm4ubmxbNkymjRp8tDlBgwYwMGDB9m9ezfZ2dk4ODgQEBDA22+/rTPFt6+vL8eOHWPTpk2kpaVhZWVFo0aNGDVq1CO38bjy8nLJyLiNnV0VTE3NSzzKzYOYmKjIz5cA/nmgKAp5eTmkpt7ExMQUU1MzQzdJCCGEEM8QI0VRytfAluXco0ahSUlJwsLCEiur0h1uSAL4509WVga5uXepVKlkHbaFeNbIKDRCiGeZjELzL5Cfn4u5uaWhmyGeARYWluTl5Rq6GUIIIYR4xhg0heZ5pFYXoFIZG7oZ4hmgUhmjVhcYuhlCCCGEKMKRG8eIiN1Oak4qduZ2vOwSQPNqjQ3dLEAC+DJRWnnv4vkm54kQQghRPh25cYx1Z0PJUxcOSHI7J5V1Z0MBykUQLyk0QgghhBBC3OOni1u1wbtGnjqPiNjtBmqRLrkDL8qFsWNHArB48bKnuqwQQgghBEDSnWSikqKJSo4mLTe9yDq3c1KfcquKJgG8eCg/v6bFqrdpUwTVq9co49YIIYQQQpSeG1lJ/wTtJ7memQBAXdtaWJpYcjf/rl79SuZ2T7uJRZIAXjzURx/N1Pl748b1JCYmMG7cRJ1yO7tKT7Sd+fMff2KtJ1lWCCGEEP8eiqKQkJVIVNJJopKjSchKBKBexTr0qR9IIwdPKltU0suBBzBVmfKyS4Chmq5DAnjxUF26dNP5e9++SNLSUvXK75ednY2FhUWxt2NqavpY7XvSZYUQQgjxfFMUhfisGxxLOklUUjSJd5IwwggXuzr0rd+TRo6e2JlX1FlG01FVRqERz62xY0eSmZnJpEnTWLRoPufOnWXgwCGMGDGK337bR0REOOfPnyM9PQ0HB0e6dQtk8OBhGBsb66wD/pfHfuzYUcaPH82sWbO5fPkSP/0USnp6Gl5ePrz//jScnWuWyrIAoaEbCQ5ey61bN3FxcWHs2AksX75UZ51CCCGEeHYoisK1zOtEJUVzPCmapLs3McKI+nb1aOf8Ij4OXlQ0f/ikm82rNaZ5tcblcjI6CeCfAQdP3yBs/yVupWVjb2tO77YutPKoZuhm6UhNvc2kSRPo3DmAgIDuVK1a2L6tW3/G0tKKfv0GYmVlyV9/HWXFim/IyspizJh3HrneVau+Q6UyZsCAIWRkpLN+/RpmzPiQ5ctXlcqy4eEhzJ8/m0aNGtOv32skJCQwdep72NjY4OAgM6QKIYQQzwpFUbiaEVeY0550kpvZKaiMVLjaudCxVht8HDyxMXvw7KbPEgngy7mDp2+wattZcvPVANxKz2HVtrMA5SqIv3kzmSlTPqJHj5465dOn/xdz8/+l0vTqFcScOZ8SHr6JN998CzMzs4euNz8/n++/X4WJSeGpamtbka++msulSxepV++FJ1o2Ly+PFSuW4uHhxYIFX2vrvfBCfWbNmi4BvBBCCFHOqRU1V9KvaUePScm+jcpIhXul+nSp0wHvKh5UMLM2dDNLnQTwT8Hv0QkcOJnwWMvGxqeRX6DolOXmq1m5NYb9x+NLtC4/7+q09qr+WO14FAsLCwICuuuV3xu837mTRW5uHj4+vmzeHMaVK39Tv77rQ9fbvfvL2sAawMenEQDx8dcfGcA/atmzZ8+QlpbG22+/olPP3z+AhQu/fOi6hRBCCGEYakXNpbQrHP8naE/NScPYyJgGlevTra4/PlUaYmVqZehmlikJ4Mu5+4P3R5UbioODo04QrHHpUizLly/l2LE/ycrK0nktKyvzkevVpOJo2NjYApCR8ehctEcte+NG4Y+q+3PiTUxMqF69bH7oCCGEEKLk1Iqa2NTLRCUX5rSn5WZgojKhQWVXerp0xdO+AVamloZu5lMjAfxT0Nrr8e98v//179xKz9Ert7c1Z/LA8tETGnTvtGtkZGQwbtxIrKwqMGLEaJycnDEzM+P8+bMsXboItVr9yPWqVMZFlivKo3/APMmyQgghhDCsAnUBFzVBe3I0GbmZmKpMaGjvTmMHLzyqNMDSpPgj3j1PJIAv53q3ddHJgQcwM1HRu62LAVtVPFFRf5GWlsasWXNo1Oh/PzYSEkqW+lNWqlUr/FEVF3cNHx9fbXl+fj4JCQm4uDw8RUcIIYQQpatAXcD51FiikqI5kXyKzLwszFSmeFRpgK+DFx727liYmBu6mQYnAXw5p+moWt5HoSmKSqUCdO945+XlER6+yVBN0uHu3pCKFSsSERFOly7dtClAu3ZtJyOj6CmUhRBCCFG68tX5nLsdS1TSSU4mnyYr/w5mxmZ42TfA19GbhvZumBs/fNCLfxsJ4J8BrTyq8ZJPDfLzH51yUp54eXljY2PLrFnTCQrqh5GRETt2bKW8ZLCYmpoyfPhI5s+fw7vvvk379h1JSEhg27YtODk5Y2RkZOgmCiGEEM+lPHU+Z1POE5UUzcmbZ7ibfxcLY3O8qjTE19GLBpXdMDOWiRofRAJ4UWYqVrRj9uz5LF68gOXLl2JjY0vnzl1p2rQ5EyeONXTzAOjTpx+KohAcvJYlS77CxaU+n3/+JQsWzMXMTB7RCSGEEKUlryCPM/8E7dE3z5BdkI2liQXeVTzwdfTCvbIrpioJTYvDSJEefSVy61YmavWDD9mNG1eoVq12qW/XxET1zN2Bf1ap1Wp69PCnbdv2TJ78YZluq6zOFyHKk/I4i6EQ4unILcjlzK1zRCUXBu05BblYm1jh7VAYtLtVegGTch60G+IaplIZYW//4EmnyvcRE6KM5eTkYG6ue6d9+/ZfSE9Pw9e3iYFaJYQQQjy7cgpyOXUzhqjkaE7fjCFXnUcFU2uaVm2Er4M3rpVcMH7ASHGieCSAF/9qJ08eZ+nSRbRr1wFb24qcP3+WX36JoF49F9q372To5gkhhBDPhOz87P8F7bfOkafOw8a0As2rN8HXwYv6dvUkaC9FEsCLf7UaNZyoUsWBkJANpKenYWtbkYCA7owePRZTU+k8I4QQQjzI3fy7RN+MISopmjMp58hX52NrZkOr6s1o7OiFi11dVEYqQzfzuSQBvPhXc3JyZvbs+YZuhhBCCPFMuJN3h5M3zxCVFM3ZlPPkKwXYmVfkpRotaeToRb2KtSVofwokgBdCCCGEEA+UmZfFyeQzRCWd5OztC6gVNZXM7Wjj/CK+jt7Usa0pQftTJgG8EEIIIYTQkZGbyYnkU0QlRXM+NRa1osbeojIdar6Er6MXtW1qynwpBiQBvBBCCCGEIC0nozBoT47mwu1YFBQcLO3pVKstvo5e1KzgJEF7OSEBvBBCCCHEv1RqThrHk09xPCmai6mXUVCoauVAlzod8HXwwqlCdQnayyEJ4IUQQggh/kVuZ6dyPPkUx5JOcjntCgoK1a2r0rVOR3wdvaluXVWC9nJOAnghhBBCiOfcrbu3OZ4cTVTSSS6nXwXAqUJ1utf1x9fRi2rWVQ3cQlESEsALIYQQQjyHbt69RVRSNFFJ0VzJuAZAzQo1CKwXgK+jF1WtHAzcQvG4ZMwf8dRt3boFP7+mJCTEa8uCggKZNWv6Yy37pI4dO4qfX1OOHTtaausUQgghDCHpTjI7/t7D539+xScHv+Cn2K0oKPRy6cb0lpOZ0vxdAup0kOD9GSd34MUjTZo0gWPH/mTLll1YWloWWWfixLGcPh1NRMROzM3Nn3ILi2f37h2kpNzi1VcHGLopQgghRKm5kZVUeKc9+STXMxMAqGtbi1de6I6vgxf2lpUN3EJR2iSAF4/k79+FP/74jQMHfsXfP0Dv9du3U/jrrz/p3LnrYwfv69aFolKV7QOhyMidXLhwXi+Ab9SoMZGRv2Nqalqm2xdCCCFKg6IoJGQlEpV0kqjkaBKyEgGoV7EOfeoH0sjBk8oWlQzcSlGWJIAXj/TSS+2wtLRi9+4dRQbwe/bspqCggM6d9V8rLjMzsydp4hNRqVTl9qmBEEIIAYVB+/XMBKKSC3PaE+8kYYQRLnZ16Fu/J40cPbEzr2joZoqnRAJ48UgWFha89FJb9u7dTXp6Ora2tjqv7969A3t7e2rWrM3cuZ/z119HSExMxMLCgsaNmzJmzDtUr17jodsICgrE17cJH3wwXVt26VIsCxbM4dSpaCpWrEjPnr2pUkU/Z++33/YRERHO+fPnSE9Pw8HBkW7dAhk8eBjGxsYAjB07kuPHjwHg59cUgGrVqhMSsoVjx44yfvxoFi78hsaNm2rXGxm5kx9//IErV/7Gysqa1q1f4q23xmNnZ6etM3bsSDIzM/n445l8+eVsYmJOY2NjS9++/Rk4cGjJDrQQQghxD0VRuJZ5naikaI4nRZN09yZGGFG/kgvtnFvj4+BJRXMbQzdTGIAE8M+AIzeOseXSdlKyU6lkbsfLLgE0r9b4qbbB3z+AnTu3sW9fJC+//Iq2/MaNBE6dOklQUH9iYk5z6tRJOnXqgoODIwkJ8fz0Uyjjxo3ixx83YWFhUezt3bp1k/HjR6NWqxk0aCgWFpZERIQXead869afsbS0ol+/gVhZWfLXX0dZseIbsrKyGDPmHQCGDh3O3bt3SUxMYNy4iQBYWlo9cPtbt27h009n4OHhxVtvjScpKZHQ0A3ExJxm+fLVOu1IT0/jP/8ZT/v2HenYsTN79+5m6dJF1Kv3Aq1atS72PgshhBCKonA1I+6f0WNOcjM7BZWRClc7FzrWaoOPgyc2ZhUM3UxhYBLAl3NHbhxj3dlQ8tR5ANzOSWXd2VCApxrEN2vWAju7SuzevUMngN+9eweKouDv3wUXlxdo376TznKtW7dh9Ohh7NsXSUBA92Jvb+3aVaSlpbJixRrc3NwB6Nq1B6+99ope3enT/4u5+f9+HPTqFcScOZ8SHr6JN998CzMzM5o1a0lY2CbS0lLp0qXbQ7edn5/P0qWLeOEFVxYt+lab3uPm5s706R+wZUs4QUH9tfWTkhL55JP/atOLevToSVBQD375ZbME8EIIIR5Jrai5kn7tn46o0aRk30ZlpMK9Un261OmAdxUPKphZG7qZohyRAP4pOJzwFwcT/nysZS+nXSVfydcpy1PnsTYmhD/ij5RoXa2qN6NF9SaP1Q4TExM6dOjETz+FcvPmTapUqQLA7t07cXauScOGnjr18/PzycrKxNm5JhUq2HD+/NkSBfAHD/6Ol5ePNngHqFSpEv7+XQkP36RT997g/c6dLHJz8/Dx8WXz5jCuXPmb+vVdS7SvZ8+e4fbtFG3wr9Ghgz9LlnzFH3/8rhPAV6hQgU6dumj/NjU1pUEDD+Ljr5dou0IIIf491IqaS2lXOP5P0J6ak4axkTENKtenW11/fKo0xMr0wU+Kxb+bBPDl3P3B+6PKy5K/fwBhYZvYs2cnr746gL//vszFi+cZNuxNAHJyslmz5ge2bt1CcnISiqJol83MzCzRthITb+Dl5aNXXqtWbb2yS5diWb58KceO/UlWVpbOa1lZJdsuFKYFFbUtlUqFs3NNEhMTdModHfWnnLaxsSU29mKJty2EEOL5pVbUxKZeJiq5MKc9LTcDE5UJDSq70tOlK572DbAyLXq4ZiHuJQH8U9CiepPHvvP94e+fcjsnVa+8krkd7zYe/aRNKxEvLx+qV3di167tvPrqAHbt2g6gTR2ZP38OW7duoW/f1/D09KJChQqAEdOnT9MJ5ktTRkYG48aNxMqqAiNGjMbJyRkzMzPOnz/L0qWLUKvVZbLde6lUxkWWl9U+CyGEeHYUqAu4qAnak6PJyM3EVGWCh707vg5eeFRpgKVJ8fuICQESwJd7L7sE6OTAA5iqTHnZ5fGHbHwSnTp1Zs2alcTFXSMycidubg20d6o1ee7jxk3Q1s/JySnx3XeAqlWrERd3Ta/86tUrOn9HRf1FWloas2bNoVGj//UJKHqmVqMiyvRVq1Zdu61716koCnFx16hb16VY6xFCCPHvVKAu4HxqLFFJJzmRfJrMvCzMVKZ4VGlQGLTbu2NhIsMXi8cnAXw5p+moauhRaDQ6d+7KmjUrWbx4PnFx13SC9aLuRIeGbqCgoKDE22nVqjWbNgVz7txZbR787du32bVrm049zeRP997tzsvL08uTB7C0tCzWjwl394ZUqlSZn34KoWvXHtoJnvbujSQ5OYmBA4eUeH+EEEI83/LV+Zy7XRi0n0w+TVb+HcyNzfC0b4Cvozce9m6YGRtuzhPxfJEA/hnQvFpjXnRuSn5+2aeDPErduvV44QVXDhzYj0qlomPH/3XefPFFP3bs2Iq1dQXq1KnL6dPRHD16hIoVSz6xxIABQ9mxYysTJ44hKKg/5uYWRESEU7VqdTIzL2jreXl5Y2Njy6xZ0wkK6oeRkRE7dmylqOwVNzd3du7cxqJFX+Lu3hBLSyv8/Nro1TMxMeGtt8bx6aczGDduFJ06dSYpKZGQkA3Uq+dCYKD+SDhCCCH+ffLU+ZxNOU9UUjQnb57hbv5dLIzN8arSEF9HbxpUdsXMWGb5FqVPAnhRYp07B3Dx4nl8fZtoR6MBeOed91CpVOzatY2cnFy8vHxYsGAJEyeOK/E2qlSpwsKF3zJ//mzWrPlBZyKnzz//P229ihXtmD17PosXL2D58qXY2NjSuXNXmjZtzsSJY3XW2bNnH86fP8vWrT+zYcM6qlWrXmQAD9CtWyBmZmasXbuKJUu+wtraGn//AEaPHieztgohxL9YXkEeZ/4J2qNvniG7IBtLEwu8q3jg6+iFe2VXTFUSXomyZaRIT7sSuXUrE7X6wYfsxo0rVKumP1LKkzIxUZWLO/CidJXV+SJEeeLgYENycoahmyHEY8styOXMrXNEJRcG7TkFuVibWOHtUBi0u1V6ARMJ2p9bhriGqVRG2Ns/eMIuOduEEEIIIe6TU5DLqZsxRCVHc/pmDLnqPCqYWtO0aiN8HbxxreSC8QNGIROirBk0gM/NzeWrr75i8+bNpKen4+7uzoQJE2jVqtVDl4uIiCAkJITY2FjS0tJwdHSkRYsWjB07FicnJ736mzZt4vvvvycuLo4aNWowZMgQBg4cWFa7JYQQQohnUHZ+9v+C9lvnyFPnYWNagebVm9DYwZsX7OpK0C7KBYMG8FOmTGHnzp0MGTKE2rVrEx4ezptvvsmaNWvw9fV94HJnz56latWqtG3blooVKxIfH8/GjRvZt28fERERODg4aOsGBwfzySefEBAQwLBhwzh69CgzZ84kJyeH4cOHP43dFEIIIUQ5dTf/LtE3Y4hKiuZMyjny1flUNLPhxRrN8HXwwsWuLiojlaGbKYQOg+XAnzx5kr59+zJ16lRef/11oHDM8B49euDo6MjatWtLtL7Tp0/Tu3dvJk2axIgRIwDIzs6mbdu2NGnShK+//lpb97333mPPnj38+uuv2NjYlGg7kgMvSpPkwIt/A8mBF+XNnbw7nLx5hqikk5xNuUC+UoCdeUV8Hbxo5OhFvYq1JWgXWpIDf4/t27djampK3759tWXm5uYEBQUxf/58kpKScHR0LPb6atSoAUB6erq27PDhw6SmpjJgwACdugMHDmTLli3s37+f7t27P+GeCCGEEKK8y8zL4mTyP0H77QuoFTWVzO1o4/wivo7e1LGtKUG7eGYYLICPiYmhbt26WFtb65R7e3ujKAoxMTGPDOBTU1MpKCggPj6eJUuWAOjkz585cwYAT09PneU8PDxQqVScOXNGAnghhBDiOZWRm8mJ5FNEJUVzPjUWtaLG3qIyHWq+RGNHb2rZOGNkVLxZuoUoTwwWwCcnJ1O1alW9ck3+elJS0iPX0aVLF1JTUwGws7Pj448/pmXLljrbMDMzw87OTmc5TVlxtiGEEEKIZ0daTkZh0J4czYXbsSgoOFja06lWW3wdvahZwUmCdvHMM1gAn52drZ2i/l6aSXJycnIeuY7Fixdz584dLl++TEREBFlZWcXahmY7xdnG/R6WjwSQlKTC2NioTC4OJibyaO95oigKKpUKB4eS9cMQ4lkk57koSyl3UzkSd5xD144Rk3wRBYUaNlV5pWEALZ0bU9tOgnbxZMrbNcxgAbyFhQV5eXl65ZqgujizXTZr1gyAtm3b0rFjRwIDA7GysmLQoEHabeTm5ha5bE5OzmPNqPmoTqxGRiru3s3GzKx0Z+uUTqzPn9zcHIyMVNK5Tzz3pBOrKAu3s1OJSo4mKimay2lXUFCobl2VrnU64uvoTXXrqoVBez7cvJlp6OaKZ5h0Yr2Hg4NDkSksycnJACXqyc/KgQAAIABJREFUwApQs2ZNPDw82LJlizaAd3BwIC8vj9TUVJ00mtzcXFJTU0u8jeKoUMGO1NRk7OwcMDU1k1/8Qo+iKOTl5ZKamoyNTSVDN0cIIZ4Zt+6mEJUczfGkaC6nXwXAqUJ1utf1x9fRi2rW+qm5QjyPDBbAu7u7s2bNGrKysnQ6sp44cUL7ekllZ2dz9+5d7d8NGjQA4NSpU/j5+WnLT506hVqt1r5emiwtC/clLe0mBQX5pbZelUqFWi134J8XxsYm2NhU0p4vQgghinbz7i2ikgrvtF/JuAZAzQo1eLleAI0cvahq5fCINQjx/DFYAB8QEMD333/Ppk2btOPA5+bmEhYWRuPGjbUdXOPj47l79y4uLi7aZVNSUqhcubLO+k6dOsXZs2fp1q2btqxly5bY2dmxbt06nQB+/fr1WFlZ0aZNmzLZN0tL61IPzOQRtBBCiH+LpDvJhUF7cjTXMq4DUNumJr1cutHIwQsHK3sDt1AIwzJYAO/j40NAQABz584lOTmZWrVqER4eTnx8PJ999pm23uTJkzly5Ajnzp3TlrVv356uXbvi6uqKlZUVFy9eJDQ0FGtra95++21tPQsLC8aPH8/MmTN555138PPz4+jRo0RERPDee+9ha2v7VPdZCCGEEEW7kZX0T9B+kuuZCQDUta3FKy90x9fBC3vLyo9YgxD/HgYL4AFmz57NggUL2Lx5M2lpabi5ubFs2TKaNGny0OUGDBjAwYMH2b17N9nZ2Tg4OBAQEMDbb79NzZo1deoOHDgQU1NTvv/+eyIjI6levToffPABQ4YMKctdE0IIIcRDKIpCQlYiUUkniUqOJiErEYB6FevQp34gjRw8qWwh/YSEKIqRoigPHlJF6HnUKDRlRVJohBDPKrl+CQ1FUbiemaAdPSbxThJGGOFiVwdfB28aOXpiZ17R0M0UQoeMQiOEEEKIfxVFUbiWeZ2opMLRY5Lu3sQII+pXcqGdc2t8HDypaF6+xtgWoryTAL6cO3j6BmG/xpKSnkNlW3N6t3WhlUc1QzdLCCGEeCBFUbiaEcexpJNEJUVzKzsFlZEKVzsXOtZqg4+DJzZmD58YUQjxYBLAl2MHT99g1baz5P4zgdOt9BxWbTsLIEG8EEKIckWtqLmSfk0btN/OSUVlpMK9Un0C6nTA28GDCqYydK4QpUEC+HIs7NdYbfCukZuvJuzXWAnghRBCGJxaUXMp7QrH/xnyMTUnDWMjYxpUrk+Pep3xrtIQK1MrQzdTiOeOBPDl2K30nBKVCyGEEGVNraiJTb2snRE1LTcDE5UJDSq70tOlK15VGmBpYmnoZgrxXJMAvhyztzUvMlg3M1ERl5SJs6PkDwohhCh7BeoCLmqC9uRoMnL/n707D4+6vPeH/54tkz2ZJDNZyDJDgOwbeyCKCrUICErlnGorx9Pax2rbY7V9DvV4nrZPz9NDq/agp1etSvH4wMFSUSCgiAhUZA1bJHuAMJOFhMxkJ9us398fMxkSk0AGksyS9+u6emm+s3zvqP3wnns+9313QyaWIiMyFXnKLGREpSFA6u/uYRJNGQzwHmztkuQhPfAAIBGLIAgCfvnuGcxNVWH1YjXilQzyREQ0vqw2Ky511KBYX4KLhnJ0m3vgJ5YhIyrNHtojU+Evlbt7mERTEgO8Bxvoc//6LjRZ0yNx8Gw9Dp2rx/kqvT3IF2gwLYqLg4iI6M5ZbBZUt19Bsb4UJYZy9Fh6IZf4ITMyDXmqbGREpsBP4ufuYRJNeTzIyUWedJBTd58Zn52pw6HzDTCZrJiXpsLqxRrEMcgTkQfhQU6ezWyzoKrtkj20t1Sgz9IHf4k/sqLSkafKQlrELPhJZO4eJpHb8CAnGlfBATJ8a0kyHpyXgM/O1OPw+QacrdRjQXo0Hl6sRmwkgzwREQ1ntppR4QjtpS0V6Lf2I0AagGxHaE+NmAWZmBGByFNxBt5FnjQD/3VdvSZ8VlSHwxcaYLbYsDA9Gg8v1iAmglt4EZH7cAbeM5isJlS0VqPYYA/tRqsJQdJAZCszkKfKQopiBqQM7UTDeOIMPAO8izw5wA/o6jXhQFEdjjiDfAxWL1YjmkGeiNyAAd59jFYTyloqUWwoRXlLJUw2M4JlQchRZiBPlY1Z4cmQiCXuHiaRR2OA9wHeEOAHdPWY8GlRLf5+4RosVgH5GfbWGpWCQZ6IJg8D/OTqt/TfDO2t1TDbzAiRBSNHlYnZymzMCNcwtBO5gAHeB3hTgB/Q2W3Ep0V1+HvxNVitAhZlxmDVYjVU4Txog4gmHgP8xOuz9KG0pRLF+lJUtFXDYrMgzC8Euaos5CmzkByugVgkdvcwibwSA7wP8MYAP6Cj24j9p2vxRXEjBMER5BepoWSQJ6IJxAA/MXrNvShpqUCxvgRVbZdhEawIl4chT5mFXFUWpoclMbQTjQMGeB/gzQF+QPsNIz49XYsvvrIH+cVZMViVr0YUgzwRTQAG+PHTbe5BiaEcxfpSVLVfhk2wQSEPR54qC3mqbKhDExjaicYZA7wP8IUAP6D9hhH7T9Xi6MVrEASgIDsWq/LViAzjcdhENH4Y4O/ODVM3LhrKUKwvxaWOGtgEGyL9IzBblY08VRYSQ+IhEoncPUwin8UA7wN8KcAPaOvqxyena3HsYiMEAbgnJw6r8pMQEcogT0R3jwHedZ3GG/bQbijF5fYaCBCgDIhEniO0JwRPY2gnmiQM8D7AFwP8gLaufnx8yh7kRSJ7kF+5kEGeiO4OA/zYdBg78ZWhDF/pS3GlQwsBAqIDlfbQrszCtOBYhnYiN2CA9wG+HOAHtHT2Yf+pWhwraYJIBNybE4eV+WooQuSTcn8i8i0M8KNr7+9AsaEUxfpSaDtrIUBAbFA08pT2nvbYoGiGdiI3Y4D3AVMhwA9o6ezDxydrcaK0CSKRCEty47BiYRKDPBG5hAF+qNa+NhQbSvGVvhTarjoAwLTgWEdoz0JMULSbR0hEgzHA+4CpFOAHGDr68PFJHU6UXodEcjPIhwczyBPR7THAAy19rSjW22faa2/UAwASguOQp8pGrioL0YFKN4+QiEbDAO8DpmKAH6Dv6MPHJ3Q4WWYP8vfnTcNDC5MQFuTn1nERkWfzhPrlDvpegyO0l6C+uxEAkBSSgDxVFnKVWVAGRrp5hEQ0FgzwPmAqB/gBze299iBffh0yiRj3z56GhxYkIZRBnohG4En1a6Jd79GjWF+CYkMprnU3AQA0oYnOE1EjAyLcPEIichUDvA9ggL+pua0X+07qcKr8OmRSMR7Ii8fyhYkIDWSQJ6KbPLF+jRdBENDU0+wM7U09zQCA6WFqzFZlI1eZCYV/uJtHSUR3gwHeBzDAD3e9rRf7TmhxuqIZMqkYS2fHY/mCRIQwyBMRPLt+3QlBEHCtu8m5e0xzrx4iiDAjXINcVRZylZkIl4e5e5hENE4Y4H0AA/zomlp7sO+EDkUVzfCTSbB0jj3IBwfI3D00InIjb6hftyMIAuq7r6FYb989Rt/XAhFEmKlIRp4yCznKTITJQ9w9TCKaAAzwPmCyA/yZ6xewt+YAOowdCJeHY3XycsyPmT1p978TjS092HtCi7OVevj5SbBsTjy+OZ9Bnmiq8tYALwgC6m404IK+BMX6UrT2t0EsEmNWeDLyVPbQHuI3+h+wROQbGOB9wGQG+DPXL+D9qo9gtpmd12RiGf4x5RHkx86blDHcjWuGbuw7qcPZSj3kfhIsmxuPB+cxyBNNNd4U4G2CDbquentPu74U7cYOiEVipCpmIk+VhWxlBoJlQe4eJhFNIgZ4HzCZAf7fT/wn2o0dIz4mEUkgl/hBLpHb/yqV3/x7iR/8JfJRHpPbH5PefO3AcyViyYT8Hg2Gbuw9ocO5Kj0C5BIsm5OAB+cnIMifQZ5oKvD0AG8TbLjaWYuv9KUoNpSiw9gJiUiCtIiZyFNlIzsqHYGyQHcPk4jcxBMDvHQSx0IuGi28A8DSxHthtJpgtBrtf7UYYbQa0W7uRb/V6LxusprGfD+pSGIP9VL5kA8H/hI5/Jyhf+iHA/9hzx/6WolYgnhlMJ57JBMN+m4UntBi30kdDp1vwDfmxuPBeQkIZJAnoklmE2yo6dA6T0TtNN2AVCxFekQK1iQ/hKyoNARIA9w9TCKiETHAezCFPHzEEK+Qh2NN8kNjeg+bYIPJahoS9vstNwP+zesjPWb/+25zj+MDggn9VuOQlp7bkYmlQ0N9ghxpcRK0tpvxadMFfL5XBrVKgVnTohDiHzDsg8NI3yKIReIx35+IaIDVZsWVgdBuKMUNUzdkYikyIlORp8xCRlQaAqT+7h4mEdFtMcB7sNXJy0fsgV+dvHzM7yEWieEv9Yf/OP6hZBNsNwO+5WbY7x8S/E1DHhsI/0aLEUaYEKIwQxzUj25jH7SWeujqbWO+v0wsu+3sv//XvkUY8lfp0A8HfvxQQOSzrDYrLnXUoFhfgouGcnSbe+AnliEjKs0e2iNT4S+Vu3uYREQuYYD3YAO7zXjaLjRikRgBUn/7TNU4/LlXe/0G9hyvwUVtMwIDgcU5KsxOVUAksYz4rYDzg4LFBJPj5z5LPzqMnUPaiSyCdcxj8HOsHXCuERh1fcGgx2/x4cFPLINIJLr7fzhE5DKLzYLq9iso1peixFCOHksv5BI/ZEWlI0+ZhfTIFPhJeE4FEXkvLmJ1EfeBnzi6610oPKbFxZpWBPlLsXxBIh6YHY8A+Z19zrTYLEO/EbAaHR8Ghl4b3B40pLVohA8ONmFs3xSIIIKfRPa1mf7h3wwMPO4nHfnDg/+gDw8yfiggLzUZ9ctss6Cq7ZI9tLdUoM/SB3+Jvz20q7KQFjELfhKutyEi13niIlYGeBcxwE88bVMXCo9rUVLTiuAAmSPIT4O/n3u/MBIEARbB6gj9X1s78LV2IuPX2olGW3fQbzFCwNj+exJBNOhbga+tERj4+xF2F7rZOjT8WwSpWMoPBTThJqp+maxmVDpCe2lLBfqt/QiQBiDbEdpTI2ZBJuYXzUR0dxjgfQAD/OSpaezE3uM6lF61B/mHFibigbx4yP0mZrtLdxAEAWabZdjC4ZsfEL7+zcAt1h0Mes1YPxSIReIR2oCGzvyPtLvQiI87PjwwMNHXjWf9MllNKG+tRrG+BGWtlTBaTQiSBiJbmYE8VRZSFDMg5X+DRDSOGOB9AAP85Ku51onC41qUadsQEijDQwuScP/saZDLfCfIjyf7hwLzsJn/W30QGPL4oLUFgx8fK286o4Amx93WL6PVhLKWShQbSlHeUgmTzYxgWRBylBnIU2VjVngy/xshognDAO8DGODd50pDJwqPX0W5rh2hgTI8tDAJ9+UxyE8G+3ak5jG2CA39FmHk3YmMMLmwHelYzigYtn5gDGcU0OS4k/rVb+m/Gdpbq2G2mRHiF4xcZRbylFmYEa7hv0MimhQM8D6AAd79Ljd0YM8xLSpr2xEW5GcP8rlx8GOQ9yoDZxT03+5bAYtp+IeDERclG2G2WcZ8/2FnFAz6VmDknYh4RsGdGmv96rP0obSlEsX6UlS0VcNisyDMLwS5KntoTw7X8J8xEU06BngfwADvOS7Vd2DPsauoqutAWLAfVjiCvEzKID9VWW1WmGymkb8VGLFl6DbtRFYTLC59KJCNaZvRqXZGwa3qV6+5FyUtFSjWl6Cq7TIsghXh8jDkKbOQq8rC9LAkn/hnQETeiwHeBzDAe57qunbsOaZFdX0HwoP9sDJfjXtzYhnkaVxYbdZbtgyNvrbg6+cWGF3ejhQYvzMKBhYeT+YZBWeuXxjxHItucw9KDOUo1peiqv0ybIINCnk48lRZmK3KRlJoAkM7EXkMBngfwADvuSpr21F47CouNXRCESLHyvwk3JMdB5mUQYA8y8AZBSOuHRjT2oLhHxwm84yCrz93pDMKzly/MOwkaYlIAlVAFJr7DLAJNkT6R2C2Kht5qiwkhsRzS1Mi8kgM8D6AAd6zCYKAqtp27DmuxWVHkF+Vn4QCBnnyYYIgDDu4bEjAH+k041HWFgxeV3A3ZxQ0djfDIgxvPxKLxFiWuAR5qiwkBE9jaCcij+eJAZ6b5ZJPEYlESFNHIDVJgYradhQe02LbwUv45HQtVuWrUZAdC6mEQZ58i0gkgkwig0wiQzCCxuU9h55RcIvTjEdpJ7II10Z8X5tgw5rkh8ZljEREU5VbA7zJZMIbb7yBwsJCdHV1ITU1FS+88ALy8/Nv+bqDBw9i//79KCkpQWtrK2JjY3H//ffjueeeQ0hIyJDnpqSkjPgev/71r/H444+P2+9CnkUkEiFDHYH0JAXKdW0oPKbF1s+q8cmpWqxalITFWQzyRLciEtlbbfwkMoRg9Fmg0fz7if9Eu7Fj2HWFPHw8hkdENKW5tYXmxRdfxMGDB7F+/XokJSVh9+7dKCsrw7Zt25CXlzfq6xYsWACVSoVly5YhLi4O1dXV2LFjB9RqNT766CPI5XLnc1NSUlBQUIDVq1cPeY+cnByo1WqXx8wWGu8kCALKtG3Yc0wLbVMXosL8sWqRGosyYxjkiSbASD3wMrEMT6R+C/NjZrtxZERErvHEFhq3BfiSkhKsW7cOL730Ep566ikAgNFoxKpVq6BSqbB9+/ZRX1tUVIQFCxYMubZnzx5s2LABGzduxNq1a53XU1JSsH79erz88svjMm4GeO8mCAJKr7ah8PhVaJtuICrMHw8vUiOfQZ5o3I22Cw0RkTfxxADvthaaAwcOQCaTYd26dc5rcrkcjz32GDZt2gS9Xg+VSjXia78e3gFg2bJlAICampoRX9Pf3w+RSDRkdp6mHpFIhOzkSGRNj0BJTSv2HNfifz6tcrTWqJGfGQ2JmEGeaDzMj5mN+TGzOQFBRDTO3JZUKisrodFoEBQ0dMFVdnY2BEFAZWWlS+/X0tICAFAoFMMe+/DDD5Gbm4vs7Gw8/PDD+Pzzz+984OQTRCIRcmZE4Zf/NBf/8q1sBMileHd/JV7eXIQTpU2w2sa+TzcRERHRZHLbDLzBYEB0dPSw60qlEgCg1+tder/NmzdDIpHgwQcfHHI9Ly8PK1asQHx8PJqamrB161b8+Mc/xh/+8AesWrXqzn8B8gkikQi5M6OQMyMSX11uQeFxLbZ8UomPT+qwerEGC9KjIRZzmzsiIiLyHG4L8P39/ZDJZMOuD7S4GI3GMb/Xvn378OGHH+KZZ55BYmLikMd27Ngx5OdHH30Uq1atwquvvoqVK1e6vAfxrfqRJppSGXL7J9Ede1AVim8s0uB02XX89WAVNn9cgf1Ftfj2N1JwT148JAzyRHeM9YuIvJmn1TC3BXh/f3+YzeZh1weC+1h71c+dO4eXX34Z9913H55//vnbPj8wMBDf/va38Yc//AFXr15FcnKyS+PmIlbfNyMmGC8/OQfFlwwoPK7FH96/gPc/q8LDi9WYn8oZeSJXsX4RkTfjItZBlErliG0yBoMBAEZdwDpYVVUVnn32WaSkpGDTpk2QSCRjundsbCwAoLOz04UR01QiFokwJ0WFvFlKXKg2oPCEFu/srcC+EzqsKdBgbqoKYp4gSURERG7gtkWsqamp0Gq16OnpGXL94sWLzsdvpa6uDk8//TQiIiLw9ttvIzAwcMz3rq+vBwBERES4OGqaasQiEeamqvD/fm8+frgmAyKRCG8VluNXW87gbJUeNvcdo0BERERTlNsC/PLly2E2m7Fz507nNZPJhF27dmH27NnOBa6NjY3DtoY0GAz43ve+B5FIhC1btowaxNva2oZda29vx/vvv4/4+Pg7OsiJpiaxSIT5adH4jSPI2wQBf95Thl+9ewbnGOSJiIhoErmthSYnJwfLly/Ha6+9BoPBgMTEROzevRuNjY3YuHGj83kbNmzAmTNnUF1d7bz29NNPo76+Hk8//TTOnz+P8+fPOx9LTEx0nuK6fft2HD58GPfddx/i4uLQ3NyMv/3tb2hra8Of/vSnyftlyWeIxfYgPzdFhTNVzdh7XIc395QhXhmMNQVq5M1SsrWGiIiIJpTbAjwAvPLKK3j99ddRWFiIzs5OpKSk4J133sGcOXNu+bqqqioAwF/+8pdhjz366KPOAJ+Xl4cLFy5g586d6OzsRGBgIHJzc/HMM8/c9h5EtyIWi7AwPQbzU6NRVNmMvSd0+NPuMiSogrGmQIO8mVEu73BERERENBYiQeB3/67gLjQ0EqvNhqIKe5DXt/chMdoe5HNnMMgTsX4RkTfjLjREPkoiFmNRZiwWpEfjdHkz9p3Q4Y8flSIpOgRrCjTImRHJIE9ERETjgjPwLuIMPI2F1WbDqbJm7DuphaGjH+oYe5DPTmaQp6mH9YuIvJknzsAzwLuIAZ5cYbHacKrsOvad1KGlsx+a2FCsKdAga3oEgzxNGaxfROTNGOB9AAM83QmL1YaTZdex74QOrV39mB4XikcKNMjQMMiT72P9IiJvxgDvAxjg6W5YrDYcL23CJyd1aO0yInlaKB4pmI50tYJBnnwW6xcReTMGeB/AAE/jwWK14XhJEz4+pUNblxEz4sOwpkCD9CQGefI9rF9E5M0Y4H0AAzyNJ7PFhuMljfj4VC3abxgxyxHkUxnkyYewfhGRN2OA9wEM8DQRzBYbvrzYiE9O6dDRbcKshHA84gjyRN6O9YuIvBkDvA9ggKeJZLZYcfSrRnxyuhad3SakJoZjTYEGKYkM8uS9WL+IyJsxwPsABniaDCazPcjvP12Lzh4T0pIUWFOgwayEcHcPjchlrF9E5M0Y4H0AAzxNJpPZii8cQb6rx4R0tT3Iz4xnkCfvwfpFRN6MAd4HMMCTOxjNVnxRfA2fnq5FV68ZGWoF1twzHTOmhbl7aES3xfpFRN6MAd4HMMCTOxlNVvy9+Bo+LarFjV4zMjURWHOPBslxDPLkuVi/iMibMcD7AAZ48gRGkxVHLjTg06I6dPeZkTU9EmsKNJgeF+ruoRENw/pFRN6MAd4HMMCTJ+k3WXD4fAM+O1OP7j4zspPtQV4TyyBPnoP1i4i8GQO8D2CAJ0/UZ7TgyIUGHCiqQ0+/BTnJkVhzjwbqGAZ5cj/WLyLyZgzwPoABnjxZn9GCQ+cbcPCMPcjnzojCmgINkmJC3D00msJYv4jIm/lsgLdYLDh8+DA6Oztx//33Q6lU3u1beiwGePIGvf0WHDpfj4Nn6tFrtCBvpj3IJ0YzyNPkY/0iIm/mEwH+lVdeQVFRET766CMAgCAIWL9+Pc6dOwdBEBAeHo4PPvgAiYmJdzdyD8UAT96kt9+Mz8814ODZevQZLZgzS4nVBRokqEYvCkTjjfWLiLyZJwZ4satveOzYMcydO9f585EjR3D27Fl8//vfxx/+8AcAwDvvvHMHQyWi8RboL8OaAg1efTYfqxerUVHbhl+9ewZ/2l2KBn23u4dHREREd0Dq6guuX7+OpKQk589///vfER8fj5///OcAgMuXL2Pfvn3jN0IiumuB/jI8cs90fGNeAg6eqcfn5+pxvtqAuakqrFmsxjQlZ+SJiIi8hcsB3mw2Qyq9+bKioiIsWrTI+XNCQgIMBsP4jI6IxlWQvwyP3usI8mfr8Pm5Bpyv0mNemgoPL9ZgWlSQu4dIREREt+FyC01MTAyKi4sB2Gfb6+vrMW/ePOfjra2tCAwMHL8REtG4Cw6QYe29yXj12UVYkZ+EizWt+OVfivD23nI0tfa4e3hERER0Cy7PwK9cuRJvvvkm2tracPnyZQQHB2PJkiXOxysrK312ASuRrwkOkOFbS5Lx4LwEHDhThyPnr+FMZTMWpEfj4UVqxEZyRp6IiMjTuBzgn3nmGTQ1NeHw4cMIDg7G73//e4SG2g+LuXHjBo4cOYKnnnpqvMdJRBMoJNAP6+6bgW/OT8RnRXU4fKEBRRXNWJgejdWLNYiO4LdqREREnmJcD3Ky2Wzo6emBv78/ZDLZeL2tR+E2kjQVdPWYcKCoDkcuNMBstSE/IwYPL1YjWsEgT65j/SIib+aJ20iOa4A3mUzw8/Mbr7fzSAzwNJV09pjw6elafFF8DRargPxMe2uNikGeXMD6RUTezBMDvMuLWI8ePYo//vGPQ65t374ds2fPRm5uLn72s5/BbDa7PlIi8jhhQX749tKZ+P0P87F0TjzOVOrxb+8U4d39ldB39Ll7eERERFOSyz3wW7ZsQWRkpPPnmpoa/Od//icSEhIQHx+P/fv3Iysri33wRD4kLFiOx5fNxEMLE7H/dC2+KG7EqbLrWJQZg4cXqREVHuDuIRIREU0ZLs/AX716FZmZmc6f9+/fD7lcjg8//BB/+ctfsGLFCuzZs2dcB0lEniE8WI4nls3C73+Yj/vypuFUeTNeeuc03vu0Ci2dnJEnIiKaDC7PwHd2dkKhUDh/PnnyJBYuXIjgYHufzvz583H06NHxGyEReRxFiBzf+cYsrFiYhE9O6fDlxUacKG3CPdmxWJmvRmSYv7uHSERE5LNcnoFXKBRobGwEAHR3d6O0tBRz5851Pm6xWGC1WsdvhETksRQhcnz3wRT87pl83JsTh2MlTfjF26ew7bNqtHX1u3t4REREPsnlGfjc3Fzs2LEDM2bMwJdffgmr1Yp7773X+XhtbS1UKtW4DpKIPFtEqD+e/GbKkBn5YyWNuCcnDisXJiEilDPyRERE48XlAP8v//IvWL9+PX76058CAB599FHMmDEDACAIAg4dOoQFCxaM7yiJyCtEhvlj/fJUrMhPwienavHlV404drERS3KmYUV+EhQhcncPkYiIyOvd0T7wHR0duHDhAkJCQjBv3jzn9c7OTuzZswcLFixAamrquA7UU3AfeKKxa+now8endDhReh0ikQj35cZhRX4SwoMZ5KcS1i8i8maeuA/8uB7kNBUwwBO5Tt/Rh49P6nCy9DokEhHuy52GFQsTEcYgPyWwfhGRN/OpAF9XV4fDhw+jvr4eAJAcuK8GAAAgAElEQVSQkIClS5ciMTHxzkbqJRjgie6cvr0X+07qcKqsGRKJCPfnTcNDC5MQFuTbJzhPdaxfROTNfCbAv/7669i8efOw3WbEYjGeeeYZPP/8866P1EswwBPdveb2Xnx8QoeT5dchk4hx/+xpeGhBEkIZ5H0S6xcReTNPDPAuL2L98MMP8dZbbyEvLw9PP/00Zs6cCQC4fPkytmzZgrfeegsJCQlYu3btnY+aiHxatCIQ31+VjlWL1Nh7QoeDZ+vx9+JreGB2PJYvSERoIIM8ERHRaFyegV+7di1kMhm2b98OqXRo/rdYLPjOd74Ds9mMXbt2jetAPQVn4InGX1NrD/ad1KGoohl+UgkemDMNy+cnIoRB3iewfhGRN/PEGXiXD3KqqanBihUrhoV3AJBKpVixYgVqampcfVsimsJiI4Pwfz2cgf/v6QXInRmFA6fr8K9/PoUPv6hBd5/Z3cMjIiLyKC630MhkMvT29o76eE9PD2Qy2V0NioimptjIIDyzOgOrFqmx74QWn56uxeELDVg2Jx7fnJ+I4ADWFiIiIpdn4LOysvC3v/0NLS0twx5rbW3FBx98gJycnHEZHBFNTdOigvDDNZn4zffnI3t6JD45VYt//fNJ7PryKnr6OSNPRERTm8s98GfPnsVTTz2FoKAgfOtb33KewnrlyhXs2rULPT09eO+99zB37twJGbC7sQeeaPI1GLqx97gW56oNCJBL8I25CXhwXgIC/Tkj7w1Yv4jIm3liD/wdbSN55MgR/Md//AeampqGXI+Li8Mvf/lL3HfffS4P1FswwBO5T73eHuTPXzIgQC7Fg/MS8I25CQj0d7kbkCYR6xcReTOfCfAAYLPZUFZWhoaGBgD2g5wyMjLwwQcfYOvWrdi/f/9t38NkMuGNN95AYWEhurq6kJqaihdeeAH5+fm3fN3Bgwexf/9+lJSUoLW1FbGxsbj//vvx3HPPISQkZNjzd+7ciXfffRcNDQ2Ii4vD+vXr8Z3vfOdOfm0GeCIPUNd8A3tP6HDhkgGBcikenG8P8gFyBnlPxPpFRN7MpwL8aP785z/jv//7v1FZWXnb57744os4ePAg1q9fj6SkJOzevRtlZWXYtm0b8vLyRn3dggULoFKpsGzZMsTFxaG6uho7duyAWq3GRx99BLn85vHsO3bswK9+9SssX74cixcvxrlz51BYWIgNGzbge9/7nsu/HwM8keeovX4De09oUXy5BUH+9hn5ZQzyHof1i4i8GQP8ICUlJVi3bh1eeuklPPXUUwAAo9GIVatWQaVSYfv27aO+tqioCAsWLBhybc+ePdiwYQM2btzoPESqv78fS5YswZw5c/Dmm286n/vzn/8cR44cwdGjR0ecsb8VBngiz1N7/QYKj2vx1RV7kP/m/EQsnRPPIO8hWL+IyJt5YoB3eRea8XLgwAHIZDKsW7fOeU0ul+Oxxx7D+fPnodfrR33t18M7ACxbtgwAhuxBX1RUhI6ODjzxxBNDnvud73wHPT09+PLLL+/21yAiD5AUE4J/eSwb/88/zUXytDDs+vIqNrx1Cp+c0qHfZHH38IiIiMaV2wJ8ZWUlNBoNgoKChlzPzs6GIAhjasEZbGBbS4VC4bxWUVEBAMjMzBzy3IyMDIjFYufjROQbNLGh+Om6HPz7+rnQxIbio6NX8a9/PoVPT9fCaLK6e3hERETjwm3fLxsMBkRHRw+7rlQqAeCWM/Aj2bx5MyQSCR588MEh9/Dz80N4ePiQ5w5cc/UeROQdpseF4oV/yEHNtU4UntBi5xc1OHCmDssXJOKBvHjI/STuHiIREdEdG1OA/5//+Z8xv+GFCxfG9Lz+/v4RT2wdWIBqNBrHfM99+/bhww8/xDPPPIPExMTb3mPgPq7cY8Ct+pEmmlLpWr8+0VSnVIZgYW48qnRteP+zKuz8ew0+P9uAtffPwEOL1PD3Y4/8ZGH9IiJv5mk1bEx/ev3+97936U1FItFtn+Pv7w+zefiJigOhevBOMrdy7tw5vPzyy7jvvvvw/PPPD7uHyWQa8XVGo3HM9xiMi1iJvE9kkAw/WZuFyw0dKDyuxbv7yvHhkctYsSAR9+VNg5+MM/ITifWLiLyZJy5iHVOA37p167gNaIBSqRyxhcVgMAAAVCrVbd+jqqoKzz77LFJSUrBp0yZIJEP/EFYqlTCbzejo6BjSRmMymdDR0TGmexCR75gZH46ffzsPl+rtQX7HkSv4tKgOKxYmYUluHIM8ERF5hTEF+Pnz54/7jVNTU7Ft2zb09PQMWch68eJF5+O3UldXh6effhoRERF4++23ERgYOOw5aWlpAICysjIUFBQ4r5eVlcFmszkfJ6KpZVZCOP7vx/NQXdeOwuNa/PXwZewvqsWKhUm4LzcOMimDPBEReS637UKzfPlymM1m7Ny503nNZDJh165dmD17tnOBa2Nj45CtIQH7LP33vvc9iEQibNmyBRERESPeY+HChQgPD8f7778/5Ppf//pXBAYG4t577x3n34qIvElKogL/+sRs/OvjeYhWBOKvhy5jw1uncPh8A8wW7lpDRESeSfLrX//61+64cUxMDK5cuYLt27ejp6cHDQ0N2LhxI2pqavDqq68iLi4OAPDcc8/hlVdewU9+8hPna5944glcvXoVjz/+OEwmE6qrq53/6+vrQ2xsLABAKpUiMDAQ7733Hq5cuYLu7m5s3boVhYWFeP7557Fo0SKXx93XZ8L4Hn01NkFBcvT2jtzPT0R3Jyo8AIuzYpCSqECDvhtffNWIE6XXIZOKEa8MhkR8+3U9NDrWLyLyZu6oYSKRCIGBfqM/Pt4nsbrCaDTi9ddfx759+9DZ2YmUlBS8+OKLQ4L1k08+iTNnzqC6utp5LSUlZdT3fPTRR/G73/1uyLUPPvgA7777LhoaGhAbG4snn3wS69evv6MxcxErkW8TBAGVte3Yc1yLKw2diAiVY2W+Gvdkx0IqcduXll6N9YuIvJknLmJ1a4D3RgzwRFODIAio0LVjz/GrqLnWhchQOVYuUqMgi0HeVaxfROTNPDHAcxNkIqIRiEQiZGgikK5WoFzbhj3Htdh6oBqfnKzFqkVJWMwgT0REbsIZeBdxBp5oahIEAWXaNuw5poW2qQtRYf5YtUiNRZkxDPK3wfpFRN7ME2fgGeBdxABPNLUJgoDSq63Yc0wL3fUbUIbfDPISMYP8SFi/iMibMcD7AAZ4IgLsQf5iTSsKj2tRe/0GVOEBeHixGgszohnkv4b1i4i8GQO8D2CAJ6LBBEHAV1daUHhci7rmbkQr7EF+QTqD/ADWLyLyZgzwPoABnohGIggCii/bg3y9vhvREYFYvcge5MVTfB951i8i8mYM8D6AAZ6IbsUmCCi+ZA/yDYZuxEQEYvViNeanTd0gz/pFRN6MAd4HMMAT0VjYBAEXqg0oPKHFNUMPYiMDsXqxBvNSVVMuyLN+EZE3Y4D3AQzwROQKmyDgfLUBe49rca2lB3FRQVi9WI25qSqIRVMjyLN+EZE3Y4D3AQzwRHQnbIKAc1V6FB7Xoqm1F9OigrC6QIM5KUqfD/KsX0TkzTwxwPMkViKiSSAWiTA/LRpzU1Q4W6XH3hNa/HlPGeKVQVi9WIPZUyDIExHR+GCAJyKaRGKxCAvSozEvVYUzlc3Ye0KHN/eUIV4ZjDUFGsyeFQURgzwREd0CAzwRkRuIxSIszIjB/LRoFFU0Y+8JLf60uxSJKnuQz53JIE9ERCNjgCciciOxWIT8zBjMT1fhdHkz9p3U4Y+7SpEY7QjyMxjkiYhoKAZ4IiIPIBGLsTgrFgszou1B/oQOf/yoFEkxIVhToEFOciSDPBERAeAuNC7jLjRENBksVhtOlV/HvhM6tHT2QxNrD/JZ070vyLN+EZE388RdaBjgXcQAT0STyWK14WTZdXx80h7kp8eFYk2BBpmaCK8J8qxfROTNGOB9AAM8EbmDxWrDidImfHxSh9YuI5IdQT7DC4I86xcReTMGeB/AAE9E7mSx2nC8tAmfOIL8jGlhWFOgQbpa4bFBnvWLiLwZA7wPYIAnIk9gttiD/McndWi/YcSM+DA8UqBBWpLnBXnWLyLyZgzwPoABnog8idliw7GSRnxyqhbtN4yYFR+GNfdMR1qSwt1Dc2L9IiJvxgDvAxjgicgTmS1WfHmxCZ+c0qGj24SUhHA8co8GKYnuD/KsX0TkzRjgfQADPBF5MrPFii++asT+U7Xo7DEhNTEcj9wzHbMSwt02JtYvIvJmDPA+gAGeiLyByewI8qdr0dVjQlqSAmsKNG4J8qxfROTNGOB9AAM8EXkTo9mKo8XX7EG+14x0tQKPFEzHjPiwSRsD6xcReTMGeB/AAE9E3shotuLvF67h06Ja3Og1I0MTgUcKNEieNvFBnvWLiLwZA7wPYIAnIm9mNFlxpLgBn56uQ3efGZnTI/BIwXRMjwudsHuyfhGRN2OA9wEM8ETkC/pNFhy5cA0HiuxBPjs5EmsKNNDEjn+QZ/0iIm/GAO8DGOCJyJf0myw4fL4BB4rq0NNvQXZyJB65RwN1zPgFedYvIvJmDPA+gAGeiHxRn9Ee5D87Yw/yuTOisKZAg6SYkLt+b9YvIvJmDPA+gAGeiHxZn9GCQ+fq8dmZevQaLcibaQ/yidF3HuRZv4jImzHA+wAGeCKaCnr7HUH+bD36jBbMnqXE6sXqOwryrF9E5M08McBLJ3EsRETkJQL9pVhdoMGyufH4/FwDDp6tw4VLBsyZpcSaAg3iVaP/wUJERBOLAZ6IiEYV6C/DmoEgf7Yen5+rx/lLBsxNUWJ1gQbxSgZ5IqLJxgBPRES3FeQvwyP3TMeyuQk4eLYeh87V43y1AXNTVVhdoMG0qCB3D5GIaMpggCciojELDpBh7b3T8eC8BHx2pg6HzjfgXJUe89JUWL1YgzgGeSKiCccAT0RELgsOkOFbS5IdQb4eh8834GylHvPTo7F6sRqxkQzyREQThbvQuIi70BARDdfVa8JnRXU4fKEBZosNC9KjkaAKxpHzDWjrMiIiVI61S5KRnxHj7qESEbnEE3ehYYB3EQM8EdHounpNOFBUh8/P1sFqG/qYn1SMf3oolSGeiLyKJwZ48SSOhYiIfFxooB/+4f4ZCA2SD3vMZLFhx6HL6DNa3DAyIiLfwR54IiIad+03jCNev9Fnxk9eP4bkaaHIUEcgQxMBdWwIJGLOJxERjRUDPBERjbvIUDlau4aH+NBAGe7JiUO5tg2Fx7XYc1yLQLkUaWqFM9ArwwPcMGIiIu/BAE9ERONu7ZJk/P+fVsFkudkI7ycV4x+XzkR+Rgy+tSQZ3X1mVOjaUK5tQ7muDeerDQAAlSLAGeZTExUI9OcfVUREg3ERq4u4iJWIaGxOlV/HrqM1Y9qFRhAEXG/rtYd5bRuq6jtgNFkhFokwfVC7jYbtNkQ0yTxxESsDvIsY4ImIXHMn9ctitaHmWifKdW0o17ZD19QFAUCAXIq0JAUyNPZAr2K7DRFNME8M8G79XtJkMuGNN95AYWEhurq6kJqaihdeeAH5+fm3fF1JSQl27dqFkpISXLp0CWazGdXV1cOe19DQgKVLl474Hps3b8a99947Lr8HERGNL6lEjJREBVISFVh7L9DdZ0ZlbTvKta0o17bhwiVHu014ANI1EchQRyAtKRyB/jI3j5yIaOK5NcD/4he/wMGDB7F+/XokJSVh9+7d+MEPfoBt27YhLy9v1NcdPXoUO3fuREpKChISEnD16tVb3mf16tUoKCgYci01NXVcfgciIpp4wQEyzEtVYV6qCoIgoLm9z9luc6r8Or4ovgaxSARNXAgy1BHI1ERCE8d2GyLyTW5roSkpKcG6devw0ksv4amnngIAGI1GrFq1CiqVCtu3bx/1tS0tLQgODoa/vz9++9vfYuvWrbecgR98j7vFFhoiItdMdP2yWG242tiFMm0bKnRt0DZ1QRCAALkEqYkKZA602ygCJ2wMROS72EIzyIEDByCTybBu3TrnNblcjsceewybNm2CXq+HSqUa8bVRUVEu36+3txdSqRR+fn53PGYiIvI8UokYsxLCMSshHGvvnY7uPjOqattR5pihL77cAgCICvN3hvm0JAXbbYjIa7ktwFdWVkKj0SAoKGjI9ezsbAiCgMrKylEDvKveeOMNbNy4ESKRCDk5Ofj5z3+OefPmjct7ExGRZwkOkGFuqgpzHe02+vY+5+z86YpmfPFVI0QiYHpsqHMxrCY2FFIJ222IyDu4LcAbDAZER0cPu65UKgEAer3+ru8hFotRUFCAb3zjG1CpVKitrcWWLVvwz//8z3jvvfcwd+7cu74HERF5LpFIhOiIQERHBGLpnHhnu025I9DvO6nD3hM6Z7vN4N1tRCKRu4dPRDQitwX4/v5+yGTDv76Uy+UA7P3wdysuLg5btmwZcm3FihVYuXIlXnvtNezYscPl97xVP9JEUypD3HZvIqK74Un1KzYmDItnJwAAuntNuHilBV9dMuBCtd7ZbqOKCETeLCXyUlTImRGF4EC2XxJNZZ5UwwA3Bnh/f3+YzeZh1weC+0CQH2/R0dFYuXIlPvjgA/T19SEgwLU9hLmIlYjINZ5ev2bFhmBWbAjW3auBvuPm7jZfFjfgs9O1EIkATezNw6Smx7Hdhmgq4SLWQZRK5YhtMgaDY2/fcep/H0lsbCxsNhu6urpcDvBEROSbRCIRohWBiFYE4oHZ9nYbbZO93aZc14aPT+mw76QO/n43220yNRFQKdhuQ0STy20BPjU1Fdu2bUNPT8+QhawXL150Pj5R6uvrIZFIEBYWNmH3ICIi7yaViDEzPhwz48PxyD3T0dvvOExK146yq6346oq93SYy1N8Z5lOTFAgO4O42RDSx3Bbgly9fjnfffRc7d+507tFuMpmwa9cuzJ4927nAtbGxEX19fUhOTnb5Hm1tbYiIiBhyrba2Fp988gnmzp0Lf3//u/49iIhoagj0l2FOigpzUuzfEOvbe1Gua0e5tg1nq5rx5UX77jbqmFBnoGe7DRFNBLcF+JycHCxfvhyvvfYaDAYDEhMTsXv3bjQ2NmLjxo3O523YsAFnzpwZclDTtWvXUFhYCAAoLS0FALz55psA7DP3DzzwAADg1VdfRX19PRYuXAiVSoW6ujrnwtUNGzZMyu9JRES+SaUIhEoRiPvzpsFqs0HbeAPlOnv//P5Ttfj4pA5yPwnSBu1uE812GyIaB24L8ADwyiuv4PXXX0dhYSE6OzuRkpKCd955B3PmzLnl6xoaGvDGG28MuTbw86OPPuoM8IsXL8aOHTvwv//7v7hx4wZCQ0OxePFi/PjHP8bMmTMn5pciIqIpRyIWY0Z8GGbEh2FNgcbRbtOBCkegv9luI0eGJgLpavv/2G5DRHdCJAjC5G+p4sW4Cw0RkWtYvwB9Rx8qHLvbVNS2o89ogQiAOjbEPjuvjkDytDC22xB5IE/chYYB3kUM8ERErmH9Gspqs0HbdAMV2jaU6dpw9VoXbIIAuUyC1MRwpDv652MiAtluQ+QBPDHAu7WFhoiIaKqRiMWYMS0MM6aFYXWBBr39FlTXtaPM0W5zsaYVABARKnfuPZ+WpEAID5MiIgcGeCIiIjcK9JfaT32dpQQAGDr6nIthz1cbcKykCSIASTE3221mxLPdhmgqYwuNi9hCQ0TkGtavO2e12aC7fsN5OmzNoHablMRw5wx9bCTbbYgmCltoiIiIaMwkYjGS48KQHBeG1Ys16DNaUFXX7gz0JY52G0XIzXabdDXbbYh8HQM8ERGRlwiQS5E3U4m8mfZ2m5ZB7TYXLhlwvNTebpMYE+IM9DOmhUEmZbsNkS9hC42L2EJDROQa1q/JYbMJjnabVnu7TWMXrDYBfjIxUhMVSHcE+ji22xC5hC00RERENCHEYhGmx4VielwoHna021TXdaDcsV1lSc1lAPZ2m3S1wnmgVCjbbYi8DgM8ERGRDwqQS5E7Mwq5M6MAAC2dfajQtaNM24avLrfgROl1AEBSdAjSNQpkqiMwIz6c7TZEXoAtNC5iCw0RkWtYvzyPzSagtvkGypy723Ta222kYsxKDEfmQLtNVBDbbWjK88QWGgZ4FzHAExG5hvXL8/UZLaiu73DubnO9rRcAEB7sN2h3mwiEBrHdhqYeTwzwbKEhIiKa4gLkUuTOiELuDHu7TWtnv3N3m6+utOBEmb3dJlEVbD9MShOBmfFhkEkl7hw20ZTFGXgXcQaeiMg1rF/ebaDdZmB2/srgdpuEcGegn8Z2G/JRnjgDzwDvIgZ4IiLXsH75ln7Tzd1tynVtaGq1t9uEfa3dJoztNuQjPDHAs4WGiIiIxszfT4qcGVHIcbTbtHX1O8N8SU0rTjrabRIGtdvMYrsN0bjiDLyLOANPROQa1q+pwyYIqBvUbnO5wd5uIxtot1FHIFMTgWlKttuQ9/DEGXgGeBcxwBMRuYb1a+rqN1lwqb4DZdo2VOja0djSAwAIC/JDuiPMp6sVCAuWu3mkRKPzxADPFhoiIiKaEP5+UmQnRyE7eVC7jWN3m9KrrThVbm+3iVcGI3PQ7jZ+MrbbEN0KZ+BdxBl4IiLXsH7RSGyCgPrmbpRpW1Gha8flhg5YrI52m/gwZGgikaGJQDzbbcjNPHEGngHeRQzwRESuYf2isTCarM7DpCp0bbjmaLcJDfJDhlrh3N0mnO02NMk8McCzhYaIiIjcTu4nQXZyJLKTIwEA7TeMzjBfpm3DqfJmAEC8Msi+u406AjMTwiFnuw1NQZyBdxFn4ImIXMP6RXdroN1moH9+oN1GKhFjVkKYM9DHq4IhZrsNjTNPnIFngHcRAzwRkWtYv2i8Gc1WXKq/eZjUNYOj3SZQhnRHmE9XR0ARwnYbunueGODZQkNEREReRS6TIGt6JLKm32y3qdDZw3yFtg2nHe0205RBztNhZ7HdhnwIZ+BdxBl4IiLXsH7RZLIJAhr03c7Z+Uv1nbBYbZBKRJgZH+5st0mIZrsNjY0nzsAzwLuIAZ6IyDWsX+RORrMVl+s7nP3zDY52m5BAmbPVJkPDdhsanScGeLbQEBERkc+SyyTInB6JzK+121Q4Av3pCke7TVSQM8ynJIRD7sd2G/JcnIF3EWfgiYhcw/pFnmqg3aZC145ybSuqv9Zuk65WIFMTyXabKc4TZ+AZ4F3EAE9E5BrWL/IWJrMVlxo6UKFtR5m2DQ2GbgBAcIDMcZCUAhnqCESE+rt5pDSZPDHAs4WGiIiICICfTIJMTSQyNZH4BwAd3Y7dbbTtKNe1ocjRbhMXFeSYnY9ASoKC7TY06TgD7yLOwBMRuYb1i3yBIAhoMPQM2t2mA2aLDRKxCDPjHYdJaSKQGB3Cdhsf44kz8AzwLmKAJyJyDesX+SKT2YrLDZ3O3W3q9TfbbQZabTI0bLfxBZ4Y4NlCQ0REROQiP5nEOeuO+4HOHpNzZ5tybRvOVOoBALGRgc4wn5IYDn8/Ri+6e5yBdxFn4ImIXMP6RVONIAi4Zuhxzs5XD2q3mTHtZrtNUnQIxGK223g6T5yBZ4B3EQM8EZFrWL9oqjNbHO02jtn5ukHtNmlJCufpsJFhbLfxRJ4Y4Pk9DhEREdEEkkklSHec+rrO0W5T6ZidL9O14WyVvd0mJiLQOTufynYbugXOwLuIM/BERK5h/SIanSAIaGzpcYb5S3UdMDnabZId7TaZbLdxK0+cgWeAdxEDPBGRa1i/iMbObLHiSkMnyhwz9HXN9nabIH8p0tT2MJ+uViAqLMDNI506PDHA87sZIiIiIg8hk0qQpo5AmjoC6+4DunpMqKi9ubvNOUe7TXREIDIH7W4TIGekm0o4A+8izsATEbmG9YtofAiCgMbWXmeYr65rv9luExfq6J+PhDqG7TbjyRNn4BngXcQAT0TkGtYvoolhtthw5drN3W1qm+3/Pwvylw7Z3SYqnO02d4MB3gcwwBMRuYb1i2hydPWaUKlrtwd6XRvabxgBANGKgEG72yjYbuMiTwzw/DdIRERE5ANCA/2wID0aC9KjIQgCmgbabXRtOF7ahCMXrkEsEiF5Wqhzdl4dGwKJWOzuoZOLOAPvIs7AExG5hvWLyP3MFhtqrnWiXNeGMm0b6q7fgAAgUC5Fmvpmu42S7TbDcAaeiIiIiCadTCpGapICqUkKfGtJMm70mlBZ244yR//8+WoDAEA10G6jtrfbBPozKnoi/lshIiIimmJCAv0wPy0a89Ps7TbX23qdYf5k6XX83dFuM31aKDLVEUjXREDDdhuP4dYWGpPJhDfeeAOFhYXo6upCamoqXnjhBeTn59/ydSUlJdi1axdKSkpw6dIlmM1mVFdXj/hcm82GLVu24K9//SsMBgPUajWeffZZrFix4o7GzBYaIiLXsH4ReReL1d5uMxDoax3tNgFyKdKTFEh3LIhVTZF2G7bQfM0vfvELHDx4EOvXr0dSUhJ2796NH/zgB9i2bRvy8vJGfd3Ro0exc+dOpKSkICEhAVevXh31uZs2bcI777yDf/zHf0RmZiYOHz6MF154AWKxGMuXL5+IX4uIiIjIa0klYqQkKpCSaG+36e4zo8JxMmy5rg3nLznabcLt7Tbp6gikJbHdZjK5bQa+pKQE69atw0svvYSnnnoKAGA0GrFq1SqoVCps37591Ne2tLQgODgY/v7++O1vf4utW7eOOAPf3NyMpUuX4vHHH8fLL78MwH4Iwne/+100NTXh0KFDELv4VRBn4ImIXMP6ReQ7BtptBvaer6rvgNFktbfbxIUiXa1ApiYSmjjfabfhDPwgBw4cgEwmw7p165zX5HI5HnvsMWzatAl6vR4qlWrE10ZFRY3pHocOHYLZbMYTTzzhvCYSifD444/jZz/7GUpKSpCbm3t3vwgRERHRFCESiRAbGYTYyCAsm5vgbLcp17WhXNuOfSd02HtChwC5BGlJEchw7E5Lc40AAA4NSURBVHCjUgS6e+g+xW0BvrKyEhqNBkFBQUOuZ2dnQxAEVFZWjhrgXblHcHAwNBrNsHsAQEVFBQM8ERER0R0a3G6z9l6gu8+Mytp2lGtbUa5twwVHu40y3B8ZanvvvL3dRubmkXs3twV4g8GA6OjoYdeVSiUAQK/Xj8s9RpqtH897EBEREZFdcIAM81JVmJeqgiAIaG7vc7bbnKpoxhdfNUIkAqbHhToDvSY2FFKJb7TbTBa3Bfj+/n7IZMM/fcnlcgD2fvjxuIefn9+43uNW/UgTTakMcdu9iYjuBusX0dSkUoUiK8U+YWux2lBd247iaj2+umTAxyft7TaB/lJkz4hC7iwV8lKUiI0MgkgkcvPIh/K0Gua2AO/v7w+z2Tzs+kCoHgjZd3sPk8k0rvfgIlYiItewfhHRAFWIH745Nx7fnBuP7j4zqgYdJnW67DoAICrM33mYVJpagSA3t9twEesgSqVyxBYWg8GxNdFd9r8P3OPcuXMTeg8iIiIicl1wgAxzU1WY62i30bf3oUzbhgpdG4oqmnF0oN0mNhTpjnab6XFstwHcGOBTU1Oxbds29PT0DFnIevHiRefjdystLQ07d+6EVqsdspB14B5paWl3fQ8iIiIiujsikQjREYGIjgjE0jnxsFhtuNrYhXJHoP/4lA77Turg7ydBaqJ9Z5tMTQRUigCPa7eZDG77CLN8+XKYzWbs3LnTec1kMmHXrl2YPXu2c4FrY2Mjampq7ugeS5cuhUwmw/vvv++8JggCduzYgbi4OOTk5NzdL0FERERE404qEWNWQjgevXc6Xl4/F//9/D147pFMLEyPRoOhG9s/v4SX3jmNDW+dwnufVuFclR7dfcNbs32V22bgc3JysHz5crz22mswGAxITEzE7t270djYiI0bNzqft2HDBpw5c2bIQU3Xrl1DYWEhAKC0tBQA8OabbwKwz9w/8MADAICYmBisX78e7777LoxGI7KysnDo0CGcO3cOmzZtcvkQJyIiIiKafEH+X2u36bi5u83ZqmZ8edHebqOOCXXOzvtyu43bTmIF7ItJX3/9dezbtw+dnZ1ISUnBiy++iEWLFjmf8+STTw4L8EVFRVi/fv2I7/noo4/id7/7nfNnm82GzZs3429/+xv0ej00Gg2eeeYZrFq16o7GzEWsRESuYf0ioolksdqgbbK325Tr2nC1sQuCAMj9JEhztNtkaCIQfYftNp64iNWtAd4bMcATEbmG9YuIJlNvv+MwKV07yq62oqWzHwAQGSp3hPlIpCUpEBwwtt1tGOB9AAM8EZFrWL+IyJ307b0o17WjXNuGyto29BmtEAFQx4Y4t6tMnhY2rN3mVPl17Dpag7YuIyJC5Vi7JBn5GTGTMmYG+HHGAE9E5BrWLyLyFFabDdrGGyjX2fvnrzZ2wSYIkPtJkJoQ7my30TZ1YeuBapgsNudr/aRi/NNDqZMS4hngxxkDPBGRa1i/iMhT2dttOlDhCPT6jj4AgFgEjBT3IkPlePW5xRM+Lo89yImIiIiIyJ0C/WWYk6LEnBQlAEDf0YcKbRu2flY94vNbu4yTObxR+ebeOkRERERELlKFB+C+vGmIDJWP+Pho1ycbAzwRERER0SBrlyTDTzo0JvtJxVi7JNlNIxqKLTRERERERIMMLFR11y40t8MAT0RERET0NfkZMcjPiPHIhfhsoSEiIiIi8iIM8EREREREXoQBnoiIiIjIizDAExERERF5EQZ4IiIiIiIvwgBPRERERORFGOCJiIiIiLwIAzwRERERkRdhgCciIiIi8iI8idVFYrFoSt6biOhusH4RkTeb7Bp2u/uJBEEQJmksRERERER0l9hCQ0RERETkRRjgiYiIiIi8CAM8EREREZEXYYAnIiIiIvIiDPBERERERF6EAZ6IiIiIyIswwBMREREReREGeCIiIiIiL8IAT0RERETkRRjgiYiIiIi8iNTdA6CR6fV6bN26FRcvXkRZWRl6e3uxdetWLFiwwN1DIyK6pZKSEuzevRtFRUVobGxEeHg48vLy8NOf/hRJSUnuHh4R0S2VlpbirbfeQkVFBVpbWxESEoLU1FT86Ec/wuzZs909PAAM8B5Lq9Vi8+bNSEpKQkpKCoqLi909JCKiMfnLX/6CCxcuYPny5UhJSYHBYMD27dvxyCOP4MMPP0RycrK7h0hENKr6+npYrVasW7cOSqUSN27cwL59+/Dd734XmzdvxuLFi909RIgEQRDcPQgarru7G2azGQqFAocOHcKPfvQjzsATkVe4cOECMjMz4efn57ym0+nw8MMPY+XKlfjd737nxtEREbmur68Py5YtQ2ZmJt5++213D4cz8J4qODjY3UMgIrojI33FrFarMXPmTNTU1LhhREREdycgIAARERHo6upy91AAcBErERFNAkEQ0NLSAoVC4e6hEBGNSXd3N9ra2nD16lX813/9Fy5duoT8/Hx3DwsAZ+CJiGgS7N27F83NzXjhhRfcPRQiojH5t3/7N3z22WcAAJlMhm9/+9v44Q9/6OZR2THAExHRhKqpqcFvfvMbzJkzB2vWrHH3cIiIxuRHP/oR/k97dxMS1f7HcfzjWNdFZaFNEGaRBYoPOC56UFHMB4gwDAqG0inSpDIDi1pEtAiKgh7ApgLLNm1yYcLALEprBKuBgigJTcKycihLNMnyqXTu4nKH5j/e/vL/XxtP837tzvd8j/M9sxg/nPmdM1arVT09PXI4HBobG9O3b9/87u8JFpbQAACmTW9vr3bv3q358+erurpaJhP/dgAYQ3x8vDIzM7V582Zdu3ZNbW1tOnLkSLDHkkSABwBMk8HBQZWXl2twcFC1tbUym83BHgkA/iezZ89WXl6eGhsbNTIyEuxxCPAAgH/f6Oio9uzZo9evX6umpkZxcXHBHgkA/i8jIyPyer36+vVrsEchwAMA/l3j4+OqqqrS06dPVV1dLYvFEuyRAGDK+vv7A2pfvnzR7du3tXjxYkVHRwdhKn/cxDqDXb58WZJ8z012OBx6/PixIiMjVVJSEszRAOAfnT59Wi6XS+vWrdPAwIAcDodv35w5c5Sfnx/E6QDg56qqqhQREaG0tDSZzWa9f/9eDQ0N6unp0fnz54M9niR+iXVGi4+Pn7QeExMjl8v1i6cBgKmx2Wx69OjRpPv4/AIw09XX18vhcKizs1OfP3/WvHnzZLFYVFpaqtWrVwd7PEkEeAAAAMBQWAMPAAAAGAgBHgAAADAQAjwAAABgIAR4AAAAwEAI8AAAAICBEOABAAAAAyHAAwAAAAZCgAcAzHg2m025ubnBHgMAZoRZwR4AABAcDx8+1Pbt2/9xf3h4uNrb23/hRACAqSDAA0CIKywsVHZ2dkDdZOJLWgCYiQjwABDiEhMTVVRUFOwxAABTxOUVAMBPeTwexcfHy263y+l0auPGjUpJSVFOTo7sdru+f/8ecExHR4f27dunNWvWKCUlRRs2bNDVq1c1Pj4e0Nvb26sTJ04oLy9PycnJSk9P186dO/XgwYOA3g8fPujgwYNatWqVUlNTVVZWpq6urmk5bwCYqbgCDwAhbnh4WP39/QH1P/74Q3PnzvVtu1wudXd3q7i4WAsXLpTL5dLFixf17t07nTp1ytf37Nkz2Ww2zZo1y9fb3Nyss2fPqqOjQ+fOnfP1ejwebd26VX19fSoqKlJycrKGh4fV2toqt9utzMxMX+/Q0JBKSkqUmpqqAwcOyOPx6Pr166qoqJDT6VR4ePg0vUMAMLMQ4AEgxNntdtnt9oB6Tk6OampqfNsdHR2qr69XUlKSJKmkpESVlZVqaGiQ1WqVxWKRJJ08eVJjY2Oqq6tTQkKCr7eqqkpOp1NbtmxRenq6JOn48eP6+PGjamtrlZWV5ff6ExMTftufPn1SWVmZysvLfbWoqCidOXNGbrc74HgA+F0R4AEgxFmtVq1fvz6gHhUV5bedkZHhC++SFBYWpl27dunOnTtqamqSxWJRX1+fnjx5ooKCAl94/7t37969unXrlpqampSenq6BgQHdu3dPWVlZk4bv/7yJ1mQyBTw1Z+3atZKkN2/eEOABhAwCPACEuGXLlikjI+O/9q1YsSKgtnLlSklSd3e3pL+WxPxY/1FcXJxMJpOv9+3bt/J6vUpMTJzSnIsWLVJERIRfbcGCBZKkgYGBKf0NAPgdcBMrAMAQfrbG3ev1/sJJACC4CPAAgCl5+fJlQK2zs1OSFBsbK0lasmSJX/1Hr1690sTEhK936dKlCgsL0/Pnz6drZAD4LRHgAQBT4na71dbW5tv2er2qra2VJOXn50uSoqOjlZaWpubmZr148cKv98qVK5KkgoICSX8tf8nOzlZLS4vcbnfA63FVHQAmxxp4AAhx7e3tcjgck+77O5hLUkJCgnbs2KHi4mKZzWbdvXtXbrdbRUVFSktL8/UdPXpUNptNxcXF2rZtm8xms5qbm3X//n0VFhb6nkAjSceOHVN7e7vKy8u1adMmJSUlaXR0VK2trYqJidHhw4en78QBwKAI8AAQ4pxOp5xO56T7GhsbfWvPc3NztXz5ctXU1Kirq0vR0dGqqKhQRUWF3zEpKSmqq6vThQsXdOPGDQ0NDSk2NlaHDh1SaWmpX29sbKxu3rypS5cuqaWlRQ6HQ5GRkUpISJDVap2eEwYAgwvz8h0lAOAnPB6P8vLyVFlZqf379wd7HAAIeayBBwAAAAyEAA8AAAAYCAEeAAAAMBDWwAMAAAAGwhV4AAAAwEAI8AAAAICBEOABAAAAAyHAAwAAAAZCgAcAAAAMhAAPAAAAGMif/wRte9EQBFsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrowCPscl3Vu"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}