EncoderBERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EncoderBERT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPKIrVrd/cChPbtApE82tnY"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "52c890e085194779ad441ee9d8cad044": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3d4672919c4d47b99b7c860702d13a5e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4a468ba3c8de40fd96742f7da7155707",
              "IPY_MODEL_b7f5cec427d54f4f8fafd7e32d6749ff",
              "IPY_MODEL_e078ba07b2594beeb8ead3a8bb332d3d"
            ]
          }
        },
        "3d4672919c4d47b99b7c860702d13a5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4a468ba3c8de40fd96742f7da7155707": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2dd4c3d747644d5fac0bb3a274c407ad",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_906054da4a904253873f86301553e951"
          }
        },
        "b7f5cec427d54f4f8fafd7e32d6749ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cbb74f72e1134677ba5722dbbde78cf6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0757c0f912b14ad9a13b0dd547887d50"
          }
        },
        "e078ba07b2594beeb8ead3a8bb332d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_472672951d964d7b8bc0043177f639cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 195B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a784bcf90b04d3db6948916fa47306c"
          }
        },
        "2dd4c3d747644d5fac0bb3a274c407ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "906054da4a904253873f86301553e951": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cbb74f72e1134677ba5722dbbde78cf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0757c0f912b14ad9a13b0dd547887d50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "472672951d964d7b8bc0043177f639cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a784bcf90b04d3db6948916fa47306c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57b61a07d2d64b81b478a279b0fc6383": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f4421563eae94c088dd37de667c7316c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7aaee56048fd445684b9dec8b7a6411e",
              "IPY_MODEL_40b6af3f37e54af0abb721b5bd11ad27",
              "IPY_MODEL_25aba31f64d348b1afb66a9b1cd4abe9"
            ]
          }
        },
        "f4421563eae94c088dd37de667c7316c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7aaee56048fd445684b9dec8b7a6411e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e0b33af1afb94720a863f90997ef7672",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fcbed9b3cb3b4bf399dc635a077967e7"
          }
        },
        "40b6af3f37e54af0abb721b5bd11ad27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5268e399a153493fb07ae428d1da1d81",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3623f23a041d437ba047180173f0cdb0"
          }
        },
        "25aba31f64d348b1afb66a9b1cd4abe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_44ed1e2ac3674f6db7647b635ff4ec8f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 226k/226k [00:00&lt;00:00, 597kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_62bb96c2e9c14cc383b59ae0eb7824ce"
          }
        },
        "e0b33af1afb94720a863f90997ef7672": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fcbed9b3cb3b4bf399dc635a077967e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5268e399a153493fb07ae428d1da1d81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3623f23a041d437ba047180173f0cdb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44ed1e2ac3674f6db7647b635ff4ec8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "62bb96c2e9c14cc383b59ae0eb7824ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "082cfc4e8a534426934a19ed9b352ecc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b76851e764a644ea8fdb975f572b9c62",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ef7ea7c617524d519a2db15c235cae0b",
              "IPY_MODEL_4c565c7fe1a243efb2ba99f7751f2153",
              "IPY_MODEL_2836257412f946cab8e6efe7a135a894"
            ]
          }
        },
        "b76851e764a644ea8fdb975f572b9c62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef7ea7c617524d519a2db15c235cae0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e8b3be37a073438c92add3e119f7a5e1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_95edc75523b84e87b86b693422aeba2f"
          }
        },
        "4c565c7fe1a243efb2ba99f7751f2153": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a4bc63bb9c9140c6bf9e66df3653d12f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_237045fd10f5414193ffa9416567a419"
          }
        },
        "2836257412f946cab8e6efe7a135a894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_df8ca0e535764192bc39c4732c3298c5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 455k/455k [00:00&lt;00:00, 632kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a0c08336037c4d9ba6f1b67972f78056"
          }
        },
        "e8b3be37a073438c92add3e119f7a5e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "95edc75523b84e87b86b693422aeba2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4bc63bb9c9140c6bf9e66df3653d12f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "237045fd10f5414193ffa9416567a419": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df8ca0e535764192bc39c4732c3298c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a0c08336037c4d9ba6f1b67972f78056": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58456a7e6fc84fbc958c1245aceb1784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d829504a53f84566a9f1174c0880a8d1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d2ce9ee1b04448aba3fbea142cefdcb6",
              "IPY_MODEL_246e51b1554f4952a88ae05b318b5a51",
              "IPY_MODEL_cdcc0e95db6a4b1e842341f9a0c6af43"
            ]
          }
        },
        "d829504a53f84566a9f1174c0880a8d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d2ce9ee1b04448aba3fbea142cefdcb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d22f3640ee7414f92309888ec419644",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4a006f1065748e4a08bd6c544963a48"
          }
        },
        "246e51b1554f4952a88ae05b318b5a51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_07719dbf516a47319c4172543df96341",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd40d0dad31840b5aaaf60bbd3821138"
          }
        },
        "cdcc0e95db6a4b1e842341f9a0c6af43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d8abd2148e6648da989358eb92c2ef4c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 5.55kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6817c01178044528d8b035415b8f8a3"
          }
        },
        "8d22f3640ee7414f92309888ec419644": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4a006f1065748e4a08bd6c544963a48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "07719dbf516a47319c4172543df96341": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fd40d0dad31840b5aaaf60bbd3821138": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8abd2148e6648da989358eb92c2ef4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6817c01178044528d8b035415b8f8a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "516675a4dd5e4d2da79ec6324dde098e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_736cecabae8c4b63b8f149dc24b3b489",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e84b55ddfff744fe9931711f966612af",
              "IPY_MODEL_d6b05dc8571a4f98a2963ba97ef86652",
              "IPY_MODEL_45f7b401b08042f181b065d51684eab2"
            ]
          }
        },
        "736cecabae8c4b63b8f149dc24b3b489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e84b55ddfff744fe9931711f966612af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_56cf6baa9bc644b2a0aacb033d1d4c18",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a3c673c3ad1460ab3732e3ddcdefd4a"
          }
        },
        "d6b05dc8571a4f98a2963ba97ef86652": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c6ddcd1b8504447843d445c004b95f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_29a4b3d965bc4415881f9733f087c669"
          }
        },
        "45f7b401b08042f181b065d51684eab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3ad22f34224747e3a7d83f94dc070302",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 420M/420M [00:22&lt;00:00, 30.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0864242456d54576b0370103475c5cc8"
          }
        },
        "56cf6baa9bc644b2a0aacb033d1d4c18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a3c673c3ad1460ab3732e3ddcdefd4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c6ddcd1b8504447843d445c004b95f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "29a4b3d965bc4415881f9733f087c669": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ad22f34224747e3a7d83f94dc070302": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0864242456d54576b0370103475c5cc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Twitter Sentiment Analysis\n",
        "\n",
        "Preprocessing of Twitter datasets, firstly with standard Machine Learning algorithms to analyse the general pattern of text. Then conceptual walkthrough of BERT from scratch with attention / encoder layers. And finally, practical application of HuggingFace transformers library with BertTokenizer and BERT model to evaluate sentiment of tweets.\n",
        "\n",
        "*And also for reference, HuggingFace's transformers library to fine-tune pretrained BERT model for a classification task.*"
      ],
      "metadata": {
        "id": "7CNm4F5rMx7C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "aZQ0KFFlMsUC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eee9HJuTNAEX",
        "outputId": "b2c4d583-10ff-4664-c60a-b95ffa6ca9dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path = \"/content/drive/My Drive/HuggingFaceTrain\""
      ],
      "metadata": {
        "id": "K49sylqVPshY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "0uDGiGChP_fe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets"
      ],
      "metadata": {
        "id": "JSGZrtXdQAie"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(path+'/Corona_NLP_train.csv', encoding='ISO-8859-1')"
      ],
      "metadata": {
        "id": "U5SVmBIhV38J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from datasets import load_dataset\n",
        "# dataset = load_dataset('csv', data_files={'train': path+'/Corona_NLP_train.csv'}, encoding = \"ISO-8859-1\")"
      ],
      "metadata": {
        "id": "B6mOWakFPtuE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "TZkDdmhDQQ1I",
        "outputId": "d8146809-2d3c-4b32-fb79-4eeb404bd89e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-6b38055f-f9e2-4e3f-b576-258b55811535\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserName</th>\n",
              "      <th>ScreenName</th>\n",
              "      <th>Location</th>\n",
              "      <th>TweetAt</th>\n",
              "      <th>OriginalTweet</th>\n",
              "      <th>Sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3799</td>\n",
              "      <td>48751</td>\n",
              "      <td>London</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3800</td>\n",
              "      <td>48752</td>\n",
              "      <td>UK</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>advice Talk to your neighbours family to excha...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3801</td>\n",
              "      <td>48753</td>\n",
              "      <td>Vagabonds</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Coronavirus Australia: Woolworths to give elde...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3802</td>\n",
              "      <td>48754</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>My food stock is not the only one which is emp...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3803</td>\n",
              "      <td>48755</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Me, ready to go at supermarket during the #COV...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41152</th>\n",
              "      <td>44951</td>\n",
              "      <td>89903</td>\n",
              "      <td>Wellington City, New Zealand</td>\n",
              "      <td>14-04-2020</td>\n",
              "      <td>Airline pilots offering to stock supermarket s...</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41153</th>\n",
              "      <td>44952</td>\n",
              "      <td>89904</td>\n",
              "      <td>NaN</td>\n",
              "      <td>14-04-2020</td>\n",
              "      <td>Response to complaint not provided citing COVI...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41154</th>\n",
              "      <td>44953</td>\n",
              "      <td>89905</td>\n",
              "      <td>NaN</td>\n",
              "      <td>14-04-2020</td>\n",
              "      <td>You know itÂs getting tough when @KameronWild...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41155</th>\n",
              "      <td>44954</td>\n",
              "      <td>89906</td>\n",
              "      <td>NaN</td>\n",
              "      <td>14-04-2020</td>\n",
              "      <td>Is it wrong that the smell of hand sanitizer i...</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41156</th>\n",
              "      <td>44955</td>\n",
              "      <td>89907</td>\n",
              "      <td>i love you so much || he/him</td>\n",
              "      <td>14-04-2020</td>\n",
              "      <td>@TartiiCat Well new/used Rift S are going for ...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>41157 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6b38055f-f9e2-4e3f-b576-258b55811535')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6b38055f-f9e2-4e3f-b576-258b55811535 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6b38055f-f9e2-4e3f-b576-258b55811535');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       UserName  ...           Sentiment\n",
              "0          3799  ...             Neutral\n",
              "1          3800  ...            Positive\n",
              "2          3801  ...            Positive\n",
              "3          3802  ...            Positive\n",
              "4          3803  ...  Extremely Negative\n",
              "...         ...  ...                 ...\n",
              "41152     44951  ...             Neutral\n",
              "41153     44952  ...  Extremely Negative\n",
              "41154     44953  ...            Positive\n",
              "41155     44954  ...             Neutral\n",
              "41156     44955  ...            Negative\n",
              "\n",
              "[41157 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert sentiment into integer and tokenize the tweets\n",
        "def transform_labels(label):\n",
        "    num = 0\n",
        "    if label == 'Positive':\n",
        "      num = 0\n",
        "    elif label == 'Negative':\n",
        "      num = 1\n",
        "    elif label == 'Neutral':\n",
        "      num = 2\n",
        "    elif label == 'Extremely Positive':\n",
        "      num = 3\n",
        "    elif label == 'Extremely Negative':\n",
        "      num = 4\n",
        "    return num"
      ],
      "metadata": {
        "id": "UsJOziizWhfk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['Sentiment_num'] = data['Sentiment'].apply(lambda x : transform_labels(x))"
      ],
      "metadata": {
        "id": "3acszQBhWqIQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = data.drop(['UserName', 'ScreenName', 'Location', 'Sentiment'], axis=1)"
      ],
      "metadata": {
        "id": "W5PgxHHCXHZz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4lI5hFSrXmk6",
        "outputId": "3218bfec-5e60-40ec-bf27-3e5787114ba6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 41157 entries, 0 to 41156\n",
            "Data columns (total 3 columns):\n",
            " #   Column         Non-Null Count  Dtype \n",
            "---  ------         --------------  ----- \n",
            " 0   TweetAt        41157 non-null  object\n",
            " 1   OriginalTweet  41157 non-null  object\n",
            " 2   Sentiment_num  41157 non-null  int64 \n",
            "dtypes: int64(1), object(2)\n",
            "memory usage: 964.7+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = data['OriginalTweet'].values\n",
        "y = data['Sentiment_num'].values\n",
        "\n",
        "X_train, X_val, y_train, y_val =\\\n",
        "    train_test_split(X, y, test_size=0.1, random_state=2020)"
      ],
      "metadata": {
        "id": "Mbmkr4qCXdu1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing\n",
        "\n",
        "In the bag-of-words model, a text is represented as the bag of its words, disregarding grammar and word order. Therefore, we will want to remove stop words, punctuations and characters that don't contribute much to the sentence's meaning. As applied else where, we will use contraction mapping and then remove stopwords / shortwords from the text."
      ],
      "metadata": {
        "id": "74oK6BO4YXIB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVtjuQLkZNP0",
        "outputId": "2838b1d5-936e-461b-f0a3-8e9d737b1056"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stopwords = nltk.corpus.stopwords.words('english')"
      ],
      "metadata": {
        "id": "1v3cGsLgZQsD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "contraction_mapping = {...}"
      ],
      "metadata": {
        "id": "FmfvkIUBYjch"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def text_cleaner(text):\n",
        "    newString = text.lower()\n",
        "    newString = re.sub('\"', '', newString)\n",
        "    newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
        "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(' ')])\n",
        "    newString = re.sub(r\"’s\\b\", \"\", newString)\n",
        "    newString = re.sub('[^a-zA-Z\\s\\-]', '', newString)\n",
        "    newString = re.sub('[^a-zA-Z\\s]', ' ', newString)\n",
        "    tokens = [w for w in newString.split() if w not in stopwords]\n",
        "    long_words = []\n",
        "    prev_word = []\n",
        "    for i in tokens:\n",
        "        if i not in prev_word and len(i) >= 3:  # removing short word\n",
        "            long_words.append(i)\n",
        "            prev_word = [i]\n",
        "    return (\" \".join(long_words)).strip()"
      ],
      "metadata": {
        "id": "YeKebGLIXqsk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "In information retrieval, TF-IDF, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus. We will use TF-IDF to vectorize our text data before feeding them to machine learning algorithms."
      ],
      "metadata": {
        "id": "sNMk_Vm9YxJr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "X_train_preprocessed = np.array([text_cleaner(text) for text in X_train])\n",
        "X_val_preprocessed = np.array([text_cleaner(text) for text in X_val])\n",
        "# Calculate TF-IDF\n",
        "tf_idf = TfidfVectorizer(ngram_range=(1, 3),\n",
        "                         binary=True,\n",
        "                         smooth_idf=False, min_df=5)\n",
        "X_train_tfidf = tf_idf.fit_transform(X_train_preprocessed)\n",
        "X_val_tfidf = tf_idf.transform(X_val_preprocessed)"
      ],
      "metadata": {
        "id": "V8yWKq77YpXp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train_tfidf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YjCDOYgZZU4o",
        "outputId": "164d9634-49e2-4445-b778-406e9354a5f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<37041x24929 sparse matrix of type '<class 'numpy.float64'>'\n",
              "\twith 758228 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Baseline with ML Algorithm - NB, SVM, etc.\n",
        "\n",
        "Dataframe of text can be approached in a similar way to a numerical dataframe, for each tweet, the words can be represented numerically. As discussed in other projects, this could be as an embedding vector to represent the general position of the words with respect to other words (GloVe, word2vec, etc.) and TF-IDF. We often result in a sparse matrix, e.g. TF-IDF gives the numerical statistic with respect to the entire vocabulary, so we would have a sparse matrix as a representation of the dataframe."
      ],
      "metadata": {
        "id": "WUdjFzs1XMWg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import TruncatedSVD\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.preprocessing import Normalizer"
      ],
      "metadata": {
        "id": "KZ59jBcD81vn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lsa = TruncatedSVD(2, algorithm = 'arpack')\n",
        "dtm_lsa = lsa.fit_transform(X_train_tfidf)\n",
        "dtm_lsa = Normalizer(copy=False).fit_transform(dtm_lsa)"
      ],
      "metadata": {
        "id": "yQIri3PQPnSd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(dtm_lsa, columns = ['component_1','component_2'])"
      ],
      "metadata": {
        "id": "3-T8BA52Pzmc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure()\n",
        "plt.scatter(df['component_1'],df['component_2'])\n",
        "plt.xlabel('First principal component')\n",
        "plt.ylabel('Second principal component')\n",
        "plt.title('Plot of points against LSA principal components')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "ze0OYON5QNhM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Multinomial Naive Bayes\n",
        "Naive Bayes methods are a set of supervised learning algorithms based on applying Bayes’ theorem with the “naive” assumption of conditional independence between every pair of features given the value of the class variable."
      ],
      "metadata": {
        "id": "6pXVkmKnZUDT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.naive_bayes import MultinomialNB"
      ],
      "metadata": {
        "id": "hQ3XlNuccx0t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = MultinomialNB()\n",
        "model.fit(X_train_tfidf, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UTG0j5arYNPS",
        "outputId": "3ab392c1-b967-4660-bb37-39a11198b1f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MultinomialNB()"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predicted_categories = model.predict(X_val_tfidf)"
      ],
      "metadata": {
        "id": "cQj8hnWoY0bD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, accuracy_score\n",
        "import seaborn as sns\n",
        "sns.set() # use seaborn plotting style"
      ],
      "metadata": {
        "id": "H4GwqEcQZDkM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# plot the confusion matrix\n",
        "mat = confusion_matrix(y_val, predicted_categories)\n",
        "sns.heatmap(mat.T, square = True, annot=True, fmt = \"d\")\n",
        "plt.xlabel(\"true labels\")\n",
        "plt.ylabel(\"predicted label\")\n",
        "plt.show()\n",
        "print(\"The accuracy is {}\".format(accuracy_score(y_val, predicted_categories)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "id": "60zI65BwYwIm",
        "outputId": "45f7637f-4973-4162-ed01-05939dba575b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUYAAAEMCAYAAAC4FB/6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXhM1xvA8e/MZJGQRZDVGmtQe6kqKhRtQ6ylaTW1tjT2LUITeyVVtYQqtcRSVCkVS2ylqCXUlsYasUQikc1kj8zM74/8OpptTJjJJJyP5z5P7j33zn1D8jr3nHvOkahUKhWCIAiCmtTQAQiCIJQ2IjEKgiDkIxKjIAhCPiIxCoIg5CMSoyAIQj4iMQqCIORjZOgAiutp/B1Dh6C1v5tMMnQIWnuoLGfoEIqlpWOcoUPQWt+YLEOHUCznY0689GcU5/fUuLLzS99P18pcYhQEoQxQKgwdwUsRiVEQBN1TKQ0dwUsRiVEQBN1TisQoCIKQh0rUGAVBEPJR5Bg6gpciEqMgCLpXxjtfxHuMgiDonkqp/VYMf/zxB7169cLd3Z2ePXty8OBBACIjIxkwYADdunVjwIAB3L17V32NprKiiMQoCILuKZXab1pSqVRMmTKFgIAAdu/eTUBAAFOnTkWpVOLn54eHhwchISF4eHjg6+urvk5TWVFEYhQEQedUKqXWm1wuJyoqqsAml8sLfK5UKiUlJQWAlJQUbG1tSUpKIjw8HDc3NwDc3NwIDw8nMTGRhISEIss0EW2MgiDoXjFqgkFBQQQGBhY47uXlxejRo9X7EomExYsXM2rUKMzNzUlLS2PVqlXExMRgZ2eHTCYDQCaTYWtrS0xMDCqVqsgyGxubImMSiVEQBN1TPNX6VE9PT3r37l3guKWlZZ79nJwcfvzxR1asWEHLli25cOEC48aNIyAg4KXDzU8kRkEQdK8YnSqWlpYFkmBhrl27RlxcHC1btgSgZcuWmJmZYWpqSmxsLAqFAplMhkKhIC4uDgcHB1QqVZFlmog2RkEQdE8PnS/29vY8evSIO3dyJ6iIiIggISGBGjVq4OLiQnBwMADBwcG4uLhgY2NDpUqViizTRFLWFsMSs+voh5hdR39ex9l1ssIOaX2uaeP3tD73999/Z/Xq1UgkEgDGjBlDly5diIiIwNvbG7lcjqWlJf7+/jg7587ao6msKCIx6pFIjPojEqP+6CQxXgnR+lzTJt1e+n669sq3MT6MiWXuwkAu/3MdY2Njur77DlPHfoGRkUx9zu79h5k+9ztmTh1Lv57dgdx3pr7/YS079uT+A/ft0Y3xI4eo/6fSJ9NaDjQ58j2Je08TMXoJxrYVqRXwJeWb1MbE3oaLrb8gO+pxgetk1hVoemIZmRHRhPearvc4WwWOwrZ9I2TmpmTGPeHW8j3c/fkY5tUq0z10KTlpmepzbwbu4fr3vwFQzr4izRYMpnKbBigysri+eBeRG47oPV4Ao2pOOP6ymrQjf5IwYwGWQz7GaojHsxOkUiQmxkR16YcyOfd1kXKtW2A9djjGNauilKeStGgl6YeO6y3Gjwb3wW3A+9Rp4EzIriPMGjc/N3ZjI+at8MOlaX0cqznwRZ/RXDh9SX2dx4iP+GhIX6xtrMhIy+DQ70dZMnsFCkXJj0JRKbXvfCmNXvnEOHdhIDYVrflj92ZSUlMZPm46W38L5tP+7gA8kafw04Zt1KlVI89123fv5+ifp9kRtByJRMLwcT44OdgzoPeHeo+51vzhpF6+rd5XKZUk/3GR6GU7aLRnQZHXVZ8+iIxbUUikJdN0fGPZbv6esApldg4V6jjSYecMksPukZ2U+57ZnnrDUCkKtiG9uXwUT/65z95hS7Cs50T7HTNIiYgh/lS43mO28R5NVvgN9b587Rbka7eo962++IxyLd5QJ0XjWtWpPH8a8b4BZJ69gLRCBaQW5fUa4+PYeNYs3kDbd1tjWs40T9mlc1f4efUv+K+aU+C64yEn+X3rPlLlqVhaW+C/eg4Dh/Vj84/b9Bpvocr47DqvfOdLVEws3VzbY2pqQuVKNrRr05KIyHvq8sUr1/NJf3esrfP2iu3efxjPj/tgb1sFuyqV8RzYl937tG83eVE27u3IeZKG/ORV9bGc+CfEBR0g9dLtIq+r0Ko+ZvWr83jbH3qP8V8pNx6izP7/ZAEqFahUlK9pq/EambkpVdo14vqSXahyFDwJv8/D4HPUHNhR7/Gad30XZUoamecuFnlO+Q/fI3XPQfW+1bBPSdmxl8y/QkGhRPlETk5UjF7j/GPfnxw/cIInSU/yHM95msOW1du5fO5qobXAh/eiSZWnArnv/CmVKqrWdNJrrEXS05DAklJiiTEpKYlr165x7do1kpKSSuq2DPqoF/uP/ElGZiaxj+M5eeY87drkdvdfDb/BP9dv8VGvDwpcFxF5j/p1njXQ1q9Ti9uR9/Uaq6yCGVUnfcz9WeuLd6FUSs15w7k7fXVugipBzRYMpueddXQ99R2Zsck8Ovzs0a77+aW8//cyWi7+AhMbCwB1U0SeJgkJWDaoptc4JeXNsR75OUmLfijyHNMWbyCzsSb9yLM2NpM3XABw2LYap5BtVJrrjdTSQq+xvoxuvbtw7OYBjoTvpV6jOuzcuNswgSgV2m+lkN4fpe/fv8/XX39NeHg4tra5tYm4uDgaNmzIrFmzqFmzpl7v37JZY379fT9vde2LQqHE/f0udO7wNgqFgjkLlzN9wkikhTx6pmdkUqHCs0cmiwrlSc/IQKVS6a2dseqUj3m89QjZMQnFus5+6Aek/n2T9Kt3MHep8fwLdOiS9zou+aynUqt6VH7bBWV2DlkJKRztNp0nYfcwqViBZgsG8+byrzj18QJy0jKJP3uDBuN7c3X2z1jWc8Lpw9ZkJRQc/qVL1iM/J3XXfhRx8UWeU8GtK+mH/0SV8axt1MiuMhU+7ELsqKkoHidQafZUbKZ4ET/jG73G+6JCfjtMyG+HqVarKh/270bi45KrhORRSmuC2tJ7jXHKlCn07duXs2fPsnfvXvbu3cvZs2fp06cPU6dO1eu9lUolX074ms4d3yb08G+c3LcNeUoqi1asZevOYOrVqUnTxi6FXmtuVo60tHT1fmpaOuZmZnpLiuaNamLZvgmPVu0p1nXGdhWxG/ohD/x/1ktcWlGqSDh3AzMHG5w9u6BIzyL5ciQqhZKseDmXpq3HrlMTjMrn9nyHfrUc8+pVeP/vZTTzH8L9HSfJiNE8dvVlGNerTbk2LZBv3lHkOZJypph36UBq8ME8x1WZ2aT+HkLO/YeoMjKRr/2Zcu+01lusuvIgMoo7N+4ydcEEwwSgh/cYS5Lea4zJycn07NkzzzGpVIq7uzs//FD0Y40uPJGnEBMbh0ffnpiYmGBiYkKvD99j2aoNVK/qyPlLV+l42kN97vWbEdy4dYfpE0dRu1YNbty+wxsN6wNw4/Yd6tSqrrdYLds2xrSaLc1CfwRAVr4cEqkUs3rVCOtW9Gs/FZrXxcS2Ik2OLQFAWs4EaTkTml9aw8UWw0v0B09qJKN8TbuCBf8+3ktz/1PJiIrn9KCF6uI3V3xF0sUIvcVVrlVTjBztqLov9z8PibkZSKUYb67Bo09GAmDeqR1KeQpZ5y/nuTb79p28zRNl6O02mZGMqjUM1MYoJqrVzNramuDgYD788EN1bUulUrFnzx6thgG9jIrWVlR1tGfbb3v5/OO+pGdksHv/YerVqcWMiV+RnZ2tPnesz1y6dnqHPm6571T17N6ZoK2/0b7tm0iQELRlJx79ehZ1q5cWt/kgCbtPqvcdRrpjWtWWSO/cRCkxNVb3NktNjZGYGqPKekry0b+51OZL9XU2PdtRuXd7bg5eoNekaFrZkirtGhFz6G8UmdnYdniDqr3bEvplIBWb1+apPJ3UO48wsS5P03mePD71DzkpGQBY1HUkIzoRRfZTqvZ8C9uOTTjUQX/vfKbu3EtayLNOKctB/TFytCdx/hL1sfJuXUkNLti5lvp7CNbDPiFt3xEUCYlYfv4xGSfO6C1WyJ3oQGYkQyqTIZNJMTE1QZGjQKFQYGxirP49MjIxxsTUhOys3J9jdw83/gw5SVJCMrXq1eTz0Z9y5tg5vcZapFJaE9SW3hPjggUL8PPzY/bs2djZ5dYmYmNjadCgAQsWFP3qia4snjeDBUt/ZO3m7UilUtq0bMqUMSOwtKiQ5zxjYyPKm5tj8f92xY96fUBU9CN6D8qtUfTt0b3QThpdUWZko8x4lqgVaZkos7LJScxte2sd+eyVi6YncmciOevYB1V2Dk8fJz+7LiUdVY4izzF9UKlU1Pq8C80ChiCRSkiPiufK1xuJOfg3VXu1pZHPAEwrW5KTkkHsn2Gc+/LZ7Cm27zahwbheyMxMSL56j1MeC8hOSNFfrJlZqDKfvWStyshElZ2NMjm311dWpRLl3mxO4oKlBa5N230AIwdb7DcsAyDjr1CSApbrLVaAoeM+Y8SkIer9D/p1Y9XCtaz6bh07Tm7GsVruON/lWxcB0OPN/sREPaLpm28wyns45uXNSEpI5vCeY6wM+EmvsRZFpSqdnSraKrGRL4mJicTE5L7m4ODg8NyxikURI1/0Q4x80Z/XceRLxrG1Wp9r9u6Q559UwkrsBW8bG5sXToaCIJQxZbxX+pUf+SIIggGINkZBEIR8RK+0IAhCPuJRWhAEIR/xKC0IgpCPSIyCIAj56OFROioqiq+++kq9n5KSQmpqKufOnSMyMhJvb2+Sk5OxtrbG399fPQ+DprKiiMQoCILu6aHzpWrVquze/Wy2oHnz5qmnX/Pz88PDwwN3d3d2796Nr68vGzZseG5ZUV75+RgFQTAAPU8ikZ2dzZ49e+jbty8JCQmEh4fj5uYGgJubG+Hh4SQmJmos00TUGAVB0L1iPErL5XLk8oLTzmlaVvXo0aPY2dnRqFEjwsLCsLOzQybLXa5EJpNha2tLTEwMKpWqyDJNA05EYhQEQfeKURMMCgoiMDCwwHEvLy9Gjx5d6DU7duygb9++Lxze84jEKAiC7hUjMXp6etK7d+8Cx4uqLcbGxhIaGkpAQACQO/dCbGwsCoUCmUyGQqEgLi4OBwcHVCpVkWWaiMQoCILuFWNuGk2PzIX57bff6NixIxUrVgSgUqVKuLi4EBwcjLu7O8HBwbi4uKgflTWVFUUkRkEQdC9Hf0MCf/vtN6ZPz7s88MyZM/H29mbFihVYWlri7++vVVlRRGIUBEH39DgkMCQkpMCx2rVrs3379kLP11RWFJEYBUHQPTHyRRAEIZ8ytDZOYcpcYhzVSr8rC+qSr9OT559USnwbV7amiRp6866hQ9DaQuu3DB1CyRM1RkEQhHxEYhQEQchLpSjbi2GJxCgIgu6JGqMgCEI+YgZvQRCEfJSiV1oQBCEv8SgtCIKQj+h8EQRByEfUGAVBEPIRbYyCIAj5iF5pQRCEfESNURAEIS+VaGMUBEHIR/RKC4Ig5CMepQVBEPLR06N0VlYW8+fP5/Tp05iamtKsWTPmzJlDZGQk3t7eJCcnY21tjb+/PzVr1gTQWFYUqV6iFwTh9aZUab8Vw7fffoupqSkhISHs2bOHsWPHAuDn54eHhwchISF4eHjg6+urvkZTWVFe6RqjkYkRn8wZjku7NyhvXYHH92PZGbCZsGOXkBkbMXzJWGo0caZyVVu+HejHzTPhea4d6DeY5l1bIzM24vb562yavprk2ET9x13NCfstP5F+9E8Sfb8BoMJHvbDw6IfUypKc+1EkLVpB9uWwfBcaYf/zKiTm5sS4DdRrjN09P6RTP1eq16/Jyd//ZPmkJeqyth+2Y8AEDyrZVyI+Jp6fAzYSevAsAB37uvLBYDccajqSkZrOid1/8nPABpSKkm2s//Gn7+jwblvMzc2Ji33M0sWr2Rj0C/Ub1OGHVd9Sq1Z1AC5dCsN78hxuXL9dovF1XDoSx3aNMDI3JeNxMld+2MvNLceo0qI2LSf1o3KTWigVSh6dvsZp341kxCUDIDUx4q1Zg6jZvRVSYxmxoTc5NW0d6Y+SSjR+fbyuk5aWxq5duzh+/DgSiQSAypUrk5CQQHh4OOvWrQPAzc2NOXPmkJiYiEqlKrJM00qBr3SNUSqTkRgTz7cD/Rjzhie7Fm7hi8AJVKpaBYBb56+xZtwykuMK/tB0HvwBzi3qMfP9SUxqM4L0J2l8PGtIicRdccoYssNvqPdNGjXAymsY8d6zeNipJ6m/76fyt7NAmvefz2LQRyiSSmbW8KTYRH5d9gtHfzmc57iNnQ1jFk8gaM4aBjUayMb56xm3dBKWlawAMDUzZf2snxjS/FOmuU/ijXZN6Dmi4JrC+vb9dytp2vBdajg2w2PAF0z3HU/TZo2IiYnF81MvalVrSe0ab7J/3xHWrF9c4vFdDvydbW3HsdFlOIcGL6Ll5H5UeqMmplblub75D7a9NZ5tbcbxNDWTDotGqK9rNLQbti3rsPO9aWxpOZqsJ2m0nfNZicdfnBqjXC4nKiqqwCaXy/N85IMHD7C2tiYwMJA+ffowaNAgzp8/T0xMDHZ2dshkMgBkMhm2trbExMRoLNPklU6M2RlZ7Fm8nYSox6hUKq4c/Zv4B3HUaOyM4mkOR9bu4/b566gKqa1UrmbHP8cvkxL/hJysp4QG/4Vj3Wp6j9nsvU4oU1LJDP1bfUzmaM/TO/d4ev0WAOl7DyGraI20onWec8q/34WUoC16jxHg7IHThB48S0py3h/eSg6VSZencfFYbvx/Hz1PZnom9jXsATi4aT/XQsPJeZpDYmwiJ3Ydp0ErlxKJ+b+uX7tFdnY2ACqVCpVKRS3nGsifpPDg/kMAJBIJSoWSWs41Sjy+5JsPUWb/f7kJVe5mWcOOqD+ucHfvOZ6mZqDIzCZ8/SFsW9VVX2dRrQoPj18lM16OIuspkb+fxbpe1RKPX5Wj0HoLCgqic+fOBbagoKA8n6lQKHjw4AENGzZk586dTJo0idGjR5Oenq7z+F/pR+n8LCpbYefsQPStB8899+S2Iwz0G4yVbUUy5Gm06dWesOMX9RqfpLw5Vl98zuNREynv/oH6eOZf57AcNACTRg3IvnaT8j27k33jFsqEZ4/1FSeN5smKNagys/Qa4/NEXLlN1O0oWnVpzd9Hz9Oyy5vkZD/l3rW7hZ7fsHUjHty8X7JB/t/C72fx8Sd9MDc34/KlfzgUckxddjfqb8pXMEcqlTJ/bsnXGAHenvc5dT9qj5GZKfFX7/Lg6KUC59i3qU/yzYfq/Ztbj/PWrEGY21mT9SSd2n3eJuqPyyUZdq5itB16enrSu3fBpwZLS8s8+w4ODhgZGeHm5gZA06ZNqVixIuXKlSM2NhaFQoFMJkOhUBAXF4eDgwMqlarIMk1em8QoM5IxbPEY/tpxnEcR0c89P+7uIxJjElh4bhWKHAUPb9znZ981eo3R6svBpP2+H0VcfJ7jqrR00o+ewPanJYAEZWoq8WOnqcvN3m0HMikZx05h2qKpXmN8HqVSyfGdfzB26URMTE3IeZrDd6P8ycoomLBdP+pC7SZ1+GHqMgNECpPG+zFl4ixat2lOu/ZtyMrKVpfVrNoCc3MzPv6kj7oGWdL+mr6e018HYduyLg5tXVBk512wrKJLNZqP782hId+rjz2JfERadAIfXwhEmaMg6foD/poRlP+j9a8YbYyWlpYFkmBhbGxsaNOmDadOneKdd94hMjKShIQEatasiYuLC8HBwbi7uxMcHIyLi4u6DVFTWVGKfJRWKpVabS+jR48eL3W9tiQSCUO+H43iaQ5btExuHrOHYmxizNimg/Fq+Cl/HzjL2CAfvcVoXK825Vq3IOXnXwuUlXf/gPI9uvFowFCi3u5Gou83VF40D2nlSkjKlcNq9AiSFwbqLbbieKNdUwZN82TmgOkMrNMH34+mMdLfi5oNa+U5782ubfCY8hnzPGeRkpRioGhzf87PnL6Ak5M9Q4Z75ClLT89g7U8/88Oqb6lcRfMvkr6olCpiQ29i7mCDy2ed1cctatrRbeNkzvhtJPbcs/bot+d9jtTUmI2NvyCo3lDu7j9Pt42TSz5wPfVKz5o1ix9//JEePXowYcIEAgICsLS0ZObMmWzatIlu3bqxadMmZs2apb5GU1lRiqwxNmzYUN3zUxiVSoVEIuHatWsab3D7dtG9eUlJJdNT5hkwEsvKViz9/BsUOdq9kV+tYU1+W7iF9CepABwN2k+viQOpUNGCVD38Ipu2bIrMwQ7H4Nw2QomZGUil2G2sQdaVf8g8eYac+1EAZJ4ORZGQgGmTRuREPcTI0R7b1f9/3DMyQlqhPI4HthM72AtFTKzOY9WkVqNahJ/9h4iruf/uEVduc+viTZq805S74ZEANOvYgi8XePHN4Nncv3GvROMrikxmpO6J/i+pVIqZuRkODvbEP9b/GwlFkRrJsKxhB0AFp0q8v8WbS4t3cXvHqTznVWpYnfMB28lOTgMgfN1BWk7uh2nFCmQlpZZYvCo9veBdrVo1Nm7cWOB47dq12b59e6HXaCorSpGJ8ciRI8X6oKK4ubnh5OSEqpAFuJOTk3VyD00+nTcchzpOLPpkDk//86gEua/k8P/kb2RshJGpMTlZTwG4eyWCtn06cvNMONkZWXT6tBtJjxL1khQB0nbuJf3gH+p9i08/wsjBnqQFiynXrg2Wgz9B9ssuFA9jMG3dEqPqVXkaEUlO1EOi//NqjmmTRlScPJpHg75EqcceaqlMisxIhlQqQyqTYmxqjCJHwe3Lt+g1sh81G9bibngktRo549K6ISEb9wHQ+O0mjF0ygYAR33D78i29xadJ5So2dOjYlpD9f5CRkcm7ndrRt78bwweP591O7UhISOKfsOuUL2/OdN/xJCc/4eaNkntdp1wlSxzbNeT+4YsoMrNxbN8YZ/e3OPbVCsztK/L+Nh/C1x/i+qajBa59fDmSOv3eIeb0NXIysnH5rAtpjxJLNCkCoGUFpLQqMjE6OTkVOKZUKomPj8fW1lbrGzg5OfHzzz9jZ2dXoKxjx45af86LsHGqTMdPuvI0K5uFoavVxzf5/MjZ3SeZc3QJlavmfi/jN34NgPc7o0iIesz2eRv4eOYQ5v6xFCMTIx7eeMCKL77VW6yqrCxUWc/a4VTpGaiys1EmPyF970GMqjpiu/I7pBYWKOIekzT/e3Lu5XYiKROe1byVT1JQKVV5julDv9ED+Gj8x+r9jn068cv3W/hlce428QdvrCtbIU+Us3P5r1w+kdtx0G/MR5hblMdn/bOXbK+HhjPP8/mPN7qiUsGQYZ+waPEcJFIpUQ8e4jN1Hvv3HcG99/v4f+eLo6M9mZlZ/H3+Mv17DcnT/lgSATb4rDNvfzMYiVRK6sN4zs7cxP1Df9N8fG8sa9rRYkIfWkzoo75kQ/1hAJyb8zNvzRlE/xMLkRobkXQjisPDDNB5VMaHBEpUhVXl8pHL5cyaNYuQkBCMjIy4dOkSR44c4cqVK4wfP17jtf7+/rz33nu0aNGiQNncuXOZMWNGsQIeXrN/sc43JN8qhnv0Kq7xceUNHUKxHEkIf/5JpcRC67cMHUKxDI3a9NKfkfJld63PtVh54KXvp2tavcfo5+dHhQoVOHr0KMbGxgA0b96c/fv3P/faqVOnFpoUgWInRUEQyoZ/3w3VZiuNtHpd5/Tp05w4cQJjY2N1h4yNjQ0JCQl6DU4QhDKqjD9Ka1VjtLCwKNCDHB0dTZUqVfQSlCAIZZyeXtcpKVolxv79+zNmzBjOnDmDUqnk4sWLTJ06lYED9TtRgSAIZZMqR6n1Vhpp9Sg9fPhwTE1NmT17Njk5Ofj4+DBgwAA8PT31HZ8gCGVR6cx3WtMqMUokEjw9PUUiFARBK/p6wbukaD1W+vTp0+zdu5e4uDhsbW358MMPadu2rT5jEwShrCrjiVGrNsa1a9cyYcIErKys6NixI9bW1kycOJG1a9fqOz5BEMoiZTG2UkirGuO6desICgqiXr166mPu7u4MHjyYIUNKZvJWQRDKjtfmUbpGjbyTdVarVk3jJBOCILy+VDllOzFqNe3Y6NGj8fHx4e7du2RmZhIZGcnXX3/NmDFjSjJWQRDKilf1Ufq/0479O2xn7969eY4FBwfTv3/ZGbssCELJ0MNaWCVK79OOCYLwGnpVE2Nh044JgiBo45WtMeZ35MgRQkNDSUpKyjMjRkBAgF4CEwSh7FLlPP+cF+Hq6oqJiQmmpqYATJo0ifbt23Pp0iV8fX3JysrCycmJb7/9lkqVKgFoLCuKVu8xBgYG4ufnh1Kp5MCBA1hbW3Py5EmtFrARBOH1o1JqvxXX0qVL2b17N7t376Z9+/YolUomT56Mr68vISEhtGrVioULFwJoLNNEqxrjjh07WLt2LfXq1WPnzp34+Pjg5ubGihUriv9dvaRdSVdK/J4vakdC2XmeGFaplaFDKJY9OYZdJrY4Rj4+ZugQimWoDj6jJB+lw8LCMDU1pVWr3J/hgQMH0rlzZ7755huNZZpolRjlcrn65W5jY2OePn1KkyZNCA0NfZnvRxCEV5VK+3ec5XI5crm8wPGillWdNGkSKpWKli1bMmHCBGJiYnB0dFSX29jYoFQqSU5O1lhmbW1dZExaJcbq1atz69Yt6tatS926ddmyZQuWlpZYWVlpc7kgCK+Z4tQYg4KCCAwsuPyvl5cXo0ePznNs8+bNODg4kJ2dzbx585g9ezbvvffey4ZbgFaJcdy4ceoV/SZOnMikSZNIT0/Hz89P5wEJglD2qZTa1xg9PT3p3bt3geOF1RYdHBwAMDExwcPDg5EjR/LZZ58RHR2tPicxMRGpVIq1tTUODg5FlmmiVWL872p+TZs25dChQ9pcJgjCa0qp0D4xFvXInF96ejoKhQILCwtUKhX79u3DxcWFxo0bk5mZyfnz52nVqhVbt26le/fcxbg0lWlSZGJ88OCBVt9UtWrVtDpPEITXhz46XxISEhg9ejQKhQKlUknt2rXx8/NDKpUSEBCAn59fnldyAI1lmhS5fGqDBg2QSCQaV/GSSCRcu3btBb/NF1PFqn6J3u9lKJSiV1pfljw6aegQtFZaV8IrSnZW1Et/xnKH3WwAACAASURBVIM3O2t9brXQ0jfKrsga4/Xr10syDkEQXiFl7P+CArQe+SIIgqCt4nS+lEYiMQqCoHPF6XwpjURiFARB50SNURAEIR9VMUa+lEYiMQqCoHOv7LRjHTt21GpNl2PHjukyHkEQXgHKV7XG+N+XIK9evcquXbsYNGgQjo6OREdHs2nTJnr16lUiQQqCULa8so/SrVu3Vn89e/Zs1qxZg52dnfpYhw4dGDZsmFg+VRCEAl6LXum4uDjMzc3zHDM3Nyc2NlYvQQmCULaV9V5prWbwdnV1ZeTIkZw6dYqIiAhOnjzJV199haurq77j0wtn5xo8iL3CilW5zQXt2rfh+F+/c/teKDciz7B+UyD2DrYGjhJWrl5I+K1T3Ht4kXMXDzLI89mKjIM8+3P+0mHux1xi+8412NuXbLwyEyP6+Y9g2smlzAlby7h931D/3aYAVKxamYC7W5jzzzr11nn0s9lTmnz4FqN2zGLutfV8sfXrEo37XyO/9OSvU3uRP7nN6tWL8pR16tSOK5f/ICnxJiEh26hevXStf9SgQR1CDmzjcVw44eEnce/5/EkRSppSJdF6K42KHCv9X1lZWSxbtowDBw4QFxeHra0t3bt3x8vLi3LlypVEnGq6GCv9y29rMCtXjgcPohk1YjJVqlRCKpMR+ygOExNjvGeMo25dZwZ9PPKl7vOyY6UbNKjDnTv3yc7Opm49Z37ft4mB/YZToUJ51m5YivsHnxIRcY9vAmZQv0Ederz/yQvfq7hjpY3NTHn3CzfO/3qc5IcJNOjUDI+lo1nUfQoA004uw7v2JygVBf8O6rRrjLl1BWxrO1L77Ub8OHBOseN92bHS7u7dUSlVvPdeR8qZlWP48AkAVKpUkWvhJ/ly5BT27j3MTL9JtGvXmg4d3V/4XrocKy2Tybhy+Q9Wrd7IsmVr6NDhLX7buZ7Wbbpx61akTu6hi7HSV2v10PrcNyL3vPT9dE2rR2lTU1MmTZrEpEmT9B2P3vXq+wHyJymEnr1ILecaADx+nJDnHKVCQS3n6oYIL4/r12+rv1apVKhUKmrVqk6LVk3Y/dt+dflC/+WE3zpFzVrVuRt5v0Rie5qRxaHFO9T7145eJPHBY5waO/Mw7I7Ga2+fCgOg9YBOeo1Rk927DwDQomUTnJwc1Md79Xqf8PCb7Ny5F4A5cxcR/fAK9evV5sbNCIPE+l8N6tfBwcGOJUtWA3Ds2F/8dTqUTzz6MnPW89cyKSllfay0Vo/SAKdOncLHx4cvv/wSyO2pPn36tN4C04cKFuXx9hnD1z4F13twqurA7XuhPIi9wqjRQwhc8pMBIizo20UziYq9wrm/DxL76DGHDh4HyPMq1b9fuzSsa5AYASpUtqKysz2xt57VNqadWobP6UD6f/sF5hUtDBZbcTR0qcfVq89mjEpPz+DOnXu4NKxnwKg0k0gkNGrUwNBh5FHWH6W1SowbN25k5syZ1KxZU73OS7ly5ViyZMlzr01KSmL69OkMGTKEzZs35ynLP225vk2bPo7NG3cQE12w0+hhVAx1arxJfee3+GbuEm7d1FzrKSmTJ8ykukMz3u86kODfD5KVlc2RQyfo1ed9GjaqT7lypkz29kKpVGJmZmaQGKVGMj5e/BUXdvzJ44ho0hJTWNpjOt+0G83SHtMxLW/Gx0u+MkhsxVW+QnmePMm7/sgTuRwLiwoGiiivGzcjiHscz8SJIzEyMqJLlw50aP8WZuYl26T1PEqlROutNNIqMQYFBbFu3TpGjBiBVJp7ibOzM5GRz2/T8PPzw8rKioEDB3L48GG8vLzIyclddFbbyXB1ofEbDejwbltWLl+v8bzkpCds+/k3NmxZgUwmK5ngnkOpVHL29AUcnewZMsyD48f+YsG8pQRtCuTSP8e4fz+K1JQ0oqMflXhsEomEgd+PQvE0h12+6wHITs8i6uodlAolqfFP2OW3jvodmmJavnT98hYmLTUNS8u8tVtLCwtSUlINFFFeOTk59O83jPe7d+bB/YuMGzeCX38N5mFUyf/ba/Ja1BjT0tLUay38+9iWk5ODsbHxc6+9e/cuU6ZMoWvXrqxdu5YqVarwxRdfkJVVsstftnunDdWqO3Hpnz/45+ZJvho9BLeeXTny584C58qMZNjaVsbCsnTUEv5lZCSjVq3cts81qzfzZvP3aFC7LXt2hyAzknEt/GaJx9QvYAQWla3Y8OX3KHMUhZ/0//YmibR0/hL8V/i1m7zRxEW9b25uhrNzDYP83Rblatg1urzXDwfHN3Bz+5RataoTev6iocPKQ6WSaL2VRlolxjfffJNVq1blObZhwwbatGnz3GufPn2q/loikeDn50e9evUYMWJEiSbHDeu30brZe3R6pxed3unF+nVbOXzwGB/1HsqHPd6jdp1aSCQSKlWqyJz507hy+R+Sk56UWHz5Va5sQ5++H1K+vDlSqRTXzu/Qp58bx4//hampCS4uue2JTlUd+H7pXH78IYgnyQWXoNSnPvOGYlvHiXVDvyUn69m/c7Vmtani7IBEIsHcugLuMz2JOP0PmSkZQG6CNDI1RmokQyJ59nVJkslkmJqaIpNJkcmk//9axu7dB2jUsD69er2Pqakp033GcfXqtVLR8fKvNxq7YGpqiplZOcaP/wJ7B1s2bNhu6LDy0HeNMTAwkPr163PzZu5/WJcuXaJnz55069aNIUOGkJDwrENVU1lRtEqMM2bM4NChQ7i6upKWlka3bt3Yv38/3t7ez722WrVqBdafnjp1Kk2bNuXu3bva3F4nMjIyiYuLV29pqelkZmaTkJCEg4Md23b+ROTDv/nz9B6USiWen3iVWGyFUQGDh3kQdv0EkQ8uMHueN9O953Fg31FMy5myau0iHjy6zOFjOwg9d5H5cxaXaHzWTpV565MuODaswdehK9XvKzZ3b0el6nYMDfJmzj9rmXAwgJysp2wes0x9bYs+7Zl/YwN95g3FuY0L829soN+C4SUa/7RpY5A/uc2UyV584tEX+ZPbTJs2hvj4RAZ+/AWzZ00h9lEYb7ZuzqDPSlf7qMcnfbl/7wIPoy7TqdM7fPCBB9nZ2YYOKw9VMbbi+ueff7h06RJOTrnvlyqVSiZPnoyvry8hISG0atWKhQsXPrdME63eY4Tc10WuXr3Kw4cPcXBwoEmTJur2Rk2Sk5ORSCSFrkF9+/Zt6tSpo83t1cSaL/oh1nzRn9dxzZdT9v20Prfdo1+1Pjc7O5tBgwbx3Xff8dlnn7Fy5UoyMzPx8fEhODgYyF0itXPnzly8eJErV64UWaaJVjXGkSNHIpFIaNKkCe+//z7NmjVDKpXi5fX8WpW1tXWhSREodlIUBKFsUBZjk8vlREVFFdjk8oJNQ0uWLKFnz55UrVpVfSwmJgZHR0f1vo2NDUqlkuTkZI1lmmj1gvfZs2cLPX7u3DltLhcE4TWjQvu2w6CgIAIDAwsc9/LyyvNK38WLFwkLCyuRgSYaE+O/7yk+ffq0wDuLDx48yJOJBUEQ/qUsRuuBp6cnvXv3LnDc0tIyz35oaCgRERF07py7NOujR48YOnQogwYNIjo6Wn1eYmIiUqkUa2trHBwciizTRGNifPQo990olUql/vpfDg4OJf6CtiAIZYOyGDVGS0vLAkmwMCNGjGDEiBHqfVdXV1auXEmdOnX45ZdfOH/+PK1atWLr1q107547sUbjxo3JzMwstEwTjYnxm29yh841b96cjz766LkfJgiCAMV7lH5ZUqmUgIAA/Pz8yMrKwsnJST3RtqYyTbRqYzQxMeH69es0aPBsPOb169e5fv26mMVbEIQCFCWQGI8ePar+ukWLFuzZU/gsPZrKiqJVr/SSJUvUI1/+ZW9vr9VYaUEQXj/F6ZUujbSqMaamplKhQt7hcRYWFoV2pwuCIJTWhKctrWqMtWvXJiQkJM+xQ4cOUbt2bb0EJQhC2aZCovVWGmlVY5w0aRIjRoxg//79VKtWjfv373P69OkC46cFQRAASulsYlrTqsbYqlUr9uzZwxtvvEFGRgZNmjQhODiYli1b6js+QRDKICUSrbfSSKsaI4CTk1Oed4gEQRCKUsQEdGVGkYnx66+/Zs6c3EWKJk+enGcq/f8KCAjQT2SCIJRZyiLyRVlRZGL87yDtGjVqlEgwgiC8GsrWfEIFFZkYv/jiC/XX2syiIwiC8K+y/rpOkYlR2xUA27Ztq7NgBEF4NZT1XukiE+P06dPz7MfFxQG58yv+O5eZnZ0dR44c0WN4giCURSUxJFCfikyM/x2HuHLlSpKTkxk7dixmZmZkZGSwdOnS507dow9laTbk1OwMQ4egtUXRfxo6hGJ5x9bl+SeVEifjrj3/pFdMWa8xavUe4/r165k4caJ63WIzMzMmTJjAunXr9BqcIAhlU1kfK61VYjQ3N+fKlSt5jl29etVgC7wLglC66XMxrJKg1QveY8aMYdiwYbi6umJvb8+jR4/4448/8PX11Xd8giCUQWX9UVqrxNirVy8aN25MSEgIcXFx1KpVi5EjR4rFrARBKFRpfUTWltZDAuvUqYOzszPx8fHY2trqMyZBEMo4RRmvMWrVxiiXy5k4cSJNmjSha9euABw5coTvv/9er8EJglA26avzZdSoUfTs2ZNevXrh4eHBtWu5Pf6RkZEMGDCAbt26MWDAAO7evau+RlNZUbRKjH5+flSoUIGjR49ibGwM5K4Ds3///mJ+W4IgvA70lRj9/f35/fff2bVrF0OGDMHHxwfIzVEeHh6EhITg4eGRp/9DU1lRtEqMp0+fZsaMGdja2qonk7CxsSEhIaGY35YgCK8DffVKW1hYqL9OTU1FIpGQkJBAeHg4bm5uALi5uREeHk5iYqLGMk20amO0sLAgKSkpT9tidHQ0VapUKea3JQjC66A4vdJyubzQZVKKWlZ1+vTpnDp1CpVKxU8//URMTAx2dnbIZDIAZDIZtra2xMTEoFKpiiyzsbEpMiatEmP//v0ZM2YM48aNQ6lUcvHiRRYtWsTAgQO1+sYFQXi9FOcROSgoiMDAwALHvby8Cl27ft68eQDs2rWLgIAAxo4d+6JhFkmrxDh8+HBMTU2ZPXs2OTk5+Pj4MGDAADw9PXUekCAIZV9xJqod6ulJ7969CxwvrLb4X7169cLX1xd7e3tiY2NRKBTIZDIUCgVxcXE4ODigUqmKLNPkuYlRoVDg4+PDnDlzRCIUBEErxXmULuqROb+0tDTkcrk6qR09ehQrKysqVaqEi4sLwcHBuLu7ExwcjIuLi/pRWVNZUZ6bGGUyGadOnSpyBm9BEIT89PGCd0ZGBmPHjiUjIwOpVIqVlRUrV65EIpEwc+ZMvL29WbFiBZaWlvj7+6uv01RWFIlKi+lqVq9eTUpKCl5eXpiYmLzcd/eSKlvWM+j9i0OelW7oELSmLEOzFoGYXUefcrIfvvRnfFPjU63PnXZv00vfT9e0amPctGkT8fHxrFu3Dhsbmzy1x2PHjukrNkEQyihlqZ0eQjtaJcZvv/1W33EIgvAKKeurBGr1gnfr1q2L3Moi59o1iIq7yg+rcxO+nV0VNm39gbAbJ4iX36RadScDR1i4Bg3qEHJgG4/jwgkPP4l7z+6GDqlIo0Z+zpnT+0hLucOanww/dLT35+78uHc5ByP24b1ocqHnfDbuU45FHablOy3Ux95160jgriUcuBXM4u3flVS4GlWsaM2v23/iSdItIm6dZeDAXoYOqYDXYj7G7OxslixZQteuXWnWrBldu3Zl8eLFZGVl6Ts+vfD/zo+Lf19V7yuVSo4cPsHngwq+M1VayGQydvy6ln37D2Nn35hRo6awfv1S6tatZejQChUdE8v8b5awbv02Q4cCQHxsAhuXbmb/tpBCyx1rOPCuW0fiH8XnOZ6SnMKvP+3k5+VbSyJMrSxbOo/s7Kc4Vm3KZ55eLF/2DQ0blq62d6VE+6000ioxzpw5kzNnzjB9+nR+/fVXpk+fzrlz55g5c6aew9O93n0/5EmynBPHny329fhxAut++pmLF65quNKwGtSvg4ODHUuWrEapVHLs2F/8dTqUTzz6Gjq0Qu3atZ/ffw8hMTHJ0KEAcGL/SU6G/IU8qeAIC4Bx88bw4/zV5DzNyXP8wsm/ORZ8nITY0jH81dzcjD69P8Bv5rekpaVz6q9Q9gQf4tNPStfPgRKV1ltppFUb45EjRzh06JD6XaM6derQtGlT9Uw7xfXkyROsrKxe6NqXUcGiPN7Tx9DL7TMGeX5U4vfXNYlEQqNGDQwdRpnX8cMOZGc95ezRc4YO5bnq1XMmJ0fBrVt31MeuXPmHDh1K12qdpTPdaU+rGmPlypXJyMi7sFNWVpZWY6WvX79Onz596NevHxEREYwYMYIOHTrQsWNH9ZRBJWXajHFs2vArMdGxJXpfXbhxM4K4x/FMnDgSIyMjunTpQIf2b2FmXs7QoZVpZuXNGO49hEC/5YYORSsVypdHLk/Jc+zJkxQsKpQ3UESFK+ttjFrVGN3d3Rk2bBiDBg3Czs6OR48esXnzZtzd3fOsP13YGtNz587lq6++IiUlhWHDhjF+/HhWrVrF0aNH8ff3Z/369Tr7ZjRp/IYLHd99m07vlL6Gam3k5OTQv98wvv9+DpMmjuLC35f59ddgsrKyDR1amfb5hM84uOMwj6LKxn+WqWlpWFpa5DlmaWlBSmqagSIqnKKM1xm1Soxbt+Y2PK9cubLA8X/LJBJJoWtMp6Wl0blzZwCWLFlCz549AXB1dWXp0qUvHnkxtWvfmmrVnbgUfgyA8uXNkclk1K9fB9cOBcdplkZXw67R5b1+6v3jx3axcdN2A0ZU9rV8pzlVHKrQ67Pcn0urSlb4rZzBlhXb2LKidHQc/dfNm3cwMpJRp04tbt+OBKBJk4aEh98wcGR5ldaaoLa0Soz/XWO6uP47sKZdu3Z5ypTKkvvr27BuG7/9ule9/9WYoVSr7sTk8X4AmJqaqKcmMjUxwdTUpNTVxt5o7MLNW3eQSiV8+aUn9g62bNhQOhOjTCbDyMgImUyKTCbD1NSUnJwcFArDvOEmk0mRGcmQyqRIZVJMTI1R5CiYMGAKRsYy9Xkr9y5n+ayVnPsjt71RKpViZCxDJpMhkUhyr1MoUeQY5vtIT8/gt137mek3iRFfTKJZ00b07NGV9h3dDRJPUUprp4q2tF7z5UU5OTmRmppKhQoVmDt3rvr4o0ePSnT51YyMTDIyMtX7aWnpZGVlkZCQ22v68HGYuuzM37mvdJS24Ycen/RlyOCBGBsbc/LUOT74wIPs7NKVvP813Wcsvl9PVO9/+klfZs/5jtlzFhkknkFjP+XzCZ+p97v2fY/1izawftGGPOcpFUpSn6SSkZ75//O64P39FHX5wYj9HPglhAUTDDfowWu0Dz+t/o6Yh1dISEjiq9HTCA+/abB4ClO206KWY6X1IT09nYyMDCpVqlSs60pbstJEjJXWHzFWWn90MVZ6bE3t52pdcrf0vCP6L73XGItibm6Oubm5oW4vCIIevRadL4IgCMUh2hgFQRDyKdtpUSRGQRD0QNQYBUEQ8inr7zFqNSRQEAShOFTF+KOtpKQkhg8fTrdu3ejRowdeXl7q9aEvXbpEz5496datG0OGDMmz5r2msqKIxCgIgs4pUGm9aUsikTBs2DBCQkLYs2cP1apVY+HChSiVSiZPnoyvry8hISG0atWKhQsXAmgs00QkRkEQdE4fk0hYW1vTpk0b9X6zZs2Ijo4mLCwMU1NTWrVqBcDAgQM5cOAAgMYyTUQboyAIOlecAQNyuRy5vOA8mZqWVVUqlWzZsgVXV1diYmJwdHRUl9nY2KBUKklOTtZYZm1tXWRMIjEKgqBzxemTDgoKIjAwsMBxLy8vRo8ufFb9OXPmYG5uzqeffsqhQ4deMMqiicQoCILOFed1HU9PT3r3LjjDVVG1RX9/f+7du8fKlSuRSqU4ODgQHR2tLk9MTEQqlWJtba2xTBORGAVB0Lni9DZremTOb9GiRYSFhbFq1Sr1GveNGzcmMzOT8+fP06pVK7Zu3Ur37t2fW6aJSIyCIOhcjh5e8L516xY//vgjNWvWZODA3EkqqlatyvLlywkICMDPz4+srCycnJzUSz5LpdIiyzQx2Ow6L0rMrqMfYnYd/XkdZ9fpV6On1uf+eu/3l76frokaoyAIOlfWR76IxCgIgs6VsQfRAspcYnySWboW/dHExMjY0CFoLSvnqaFDKJay9HjaoGI1Q4dQ4sQkEoIgCPmIiWoFQRDyETVGQRCEfEQboyAIQj6iV1oQBCGf4ox8KY1EYhQEQedEG6MgCEI+ClXZfpgWiVEQBJ0Tj9KCIAj5lLWx9/mJxCgIgs6V7bQoEqMgCHogOl8EQRDyEYlREAQhH9ErLQiCkE9Z75UW60oLgqBzKpVK601b/v7+uLq6Ur9+fW7evKk+HhkZyYABA+jWrRsDBgzg7t27WpVpIhKjIAg6p0Sl9aatzp07s3nzZpycnPIc9/Pzw8PDg5CQEDw8PPD19dWqTJPXPjEePrSdFHkESYk3SUq8SVjYn4YOCQATExNW/ODPtesneRQbxukz++ja9V0A3nyzOXv2bORB1CXu3rvAxk3LsbevYtiA8xk18nPOnN5HWsod1vz0vaHD0agsxOpctyZrdgRy+tZh9p3ZTuf3O6rLypmZMmPBZE6EH+D0rcOs3/WDASPNVZwao1wuJyoqqsAml8vzfGarVq1wcHDIcywhIYHw8HDc3NwAcHNzIzw8nMTERI1lzyPaGIGxY2ewdt0WQ4eRh5GRjKioGLp1HciDBw/p1r0TGzYG0vrN7lSsaMXatVs4fHgkOTk5LPp+Nit/XEgvd09Dh60WHRPL/G+W0PW9dzEzK2focDQq7bHKZDKWBgXwS9BvDO8/hlZvNydw40L6d/6Me3ceMHPhNGRGMnq2H8iTJDkNGtc1dMgoijG/TlBQEIGBgQWOe3l5MXr0aI3XxsTEYGdnh0wmA3L/rmxtbYmJiUGlUhVZZmNjo/FzRWIspdLTM5g/b7F6/8D+o9y7+4DmzRuze/eBPOf+uDKIAyHbSjpEjXbt2g9Aq5ZNcXJyeM7ZhlXaY61Vtwa29pXZ8GPuf97nTl7g0rkr9Oj/Pnt3hPBut/Z0btaDtNTcVSnDr9wwZLhA8Ua+eHp60rt37wLHtV1rWh8M8ij9119/GeK2RZo7dxox0Vc5fmwXHTq0NXQ4hbK1rUydus5cu3arQFm7dm0KPS68uiQSCXUbOPNG84ZER8Xw1ZThnAg/wM5jm+jyYSdDh4eqGH8sLS2pWrVqgU2bxOjg4EBsbCwKhQIAhUJBXFwcDg4OGsueR++J8fbt2wW2adOmERERwe3bt/V9++fy8ZlPvfptqVGzJT+t2cyu39bj7FzD0GHlYWRkxNq1i9m8eQc3b0bkKWvcuAHe08Yw3We+gaIT9O3u7XskxCcx+KtPMTKS8XbH1rRq25xyZuWwc7SlnksdUuVpdGrqxvxp3zF/2dc4161p0JiVKpXW28uoVKkSLi4uBAcHAxAcHIyLiws2NjYay55HotLzHOQNGjTAyckpT7d8bGwsdnZ2SCQSjhw5UqzPMzZxev5JLyF4zyb27z/C8hXrXvqzdLFKoEQiYd36pVhaVuCj/sPJyclRlzk71yDk4DZ8v/Zny5bfXuo++lolcPasKTg5OTB02Hi9fL4u6StWXawSWK9hHabNm0DdBrX55/I1khKSyc7O5tb1O0yY8RWtar6rrhkFblzImT/PsWn1Ly90r7DYMy8dbwPbN7U+93pcqFbnzZ07l4MHDxIfH0/FihWxtrZm7969RERE4O3tjVwux9LSEn9/f5ydnQE0lmmi9zZGLy8vLl++zKxZs3B0dATA1dWVo0eP6vvWL0SlUiGRSAwdhtoPKwOwta1Mn96f50mK1ao5Ebx3MwsWLHvppCiUfjfDbzO49yj1/qbgVezeto/7d6MKnFsa1lvRx+w6M2bMYMaMGQWO165dm+3btxd6jaYyTfT+KO3l5cX48eOZMGECW7bkNh6XlsRjZWXJe+91xNTUFJlMxscf96Z9+7cIOXjM0KEBsGTpPOrXr0P/fkPJzMxSH3dwtGPf/p/5cWUQa37abMAIiyaTyf7/9yr9z9cyQ4dVqLIQa72GdTAxNaGcmSmfj/Sgsl1ldm3by4XTF4l5+IhhYz5DJpPR/M0mtG7XglN/nDVovAqVUuutNNL7o/S/srOzWbp0KWFhYdy5c4c//3yx9wV1+ShdubINe37fSP36dVAoFNy4EYHfzACOHDmhk89/mUfpatWcuH7jFJmZWXlqimNG++BcuyYzZownNTUtzzV2to1e+H66fpT2/XoCvl9PzHNs9pzvmD1nkU7vowv6jlUXj9ITfb3o80lPjI2NuHDmMvN9vuPB/2uLtevXYtYiH+o1rEPMg0cs/WYlR/Yff+F76eJR2rlyc63PvRN/8aXvp2sllhj/denSJc6dO8eIESNe6Hp9tzHqki7aGEuKvtoYBd0kxpKki8RYq1JTrc+NTLj80vfTtRJ/j7FZs2Y0a9aspG8rCEIJEtOOCYIg5FMaOoBehkiMgiDonKgxCoIg5KNQls7eZm2JxCgIgs6V9YlqRWIUBEHnRBujIAhCPqKNURAEIR9RYxQEQchHdL4IgiDkIx6lBUEQ8hGP0oIgCPnoY9qxkiQSoyAIOifeYxQEQcinrNcYX/t1pQVB0D2lSqn1VhyRkZEMGDCAbt26MWDAAO7evauX+EViFARB51QqldZbcfj5+eHh4UFISAgeHh74+vrqJX6RGAVB0LniJEa5XE5UVFSBTS6X5/nMhIQEwsPDcXNzA8DNzY3w8HASExN1Hn+Za2N8mv3Q0CEIgvAcxfk9XbZsGYGBgQWOe3l5MXr0aPV+TEwMdnZ26vV4ZDIZtra2xMTEaLUkanGUucQom26H4gAAB1VJREFUCMKrxdPTk969exc4bmlpaYBoconEKAiCQVlaWmqVBB0cHIiNjUWhUCCTyVAoFMTFxeHg4KDzmEQboyAIZUKlSpVwcXEhODgYgODgYFxcXHT+GA0GWCVQEAThRUVERODt7Y1cLsfS0hJ/f3+cnZ11fh+RGAVBEPIRj9KCIAj5iMQoCIKQj0iMgiAI+YjEKAiCkM9rnxhLalC6Lvj7++Pq6kr9+vW5efOmocPRKCkpieHDh9OtWzd69OiBl5eXXoZu6cqoUaPo2bMnvXr1wsPDg2vXrhk6pOcKDAwsEz8LZZLqNTdo0CDVrl27VCqVSrVr1y7VoEGDDBxR0UJDQ1XR0dGqTp06qW7cuGHocDRKSkpSnTlzRr2/YMEC1bRp0wwYkWZyuVz99aFDh1S9evUyYDTPFxYWpho6dGiZ+Fkoi17rGmNJDkrXhVatWunlLX99sLa2pk2bNur9Zs2aER0dbcCINLOwsFB/nZqaikQiMWA0mmVnZzN79mxmzpxp6FBeWa/1kMCSHJT+OlMqlWzZsgVXV1dDh6LR9OnTOXXqFCqVip9++snQ4RRpyZIl9OzZk6pVqxo6lFfWa11jFErGnDlzMDc359NPPzV0KBrNmzePY8eOMX78eAICAgwdTqEuXrxIWFgYHh4ehg7llfZaJ8b/DkoH9Doo/XXl7+/PvXv3WLx4MVJp2fhx69WrF2fPniUpKcnQoRQQGhpKREQEnTt3xtXVlUePHjF06FBOnjxp6NBeKWXjJ1VPSnJQ+uto0aJFhIWFsXz5ckxMTAwdTpHS0tKIiYlR7x89ehQrKyusra0NGFXhRowYwcmTJzl69ChHjx7F3t6eNWvW8M477xg6tFfKaz9WuqQGpevC3LlzOXjwIPHx8VSsWBFra2v27t1r6LAKdevWLdzc3KhZsyblypUDoGrVqixfvtzAkRUUHx/PqFGjyMjIQCqVYmVlxdSpU2nUqJGhQ3suV1dXVq5cSb169QwdyivltU+MgiAI+b3Wj9KCIAiFEYlREAQhH5EYBUEQ8hGJURAEIR+RGAVBEPIRiVHQubNnz9KhQwetzt25cycff/zxC93nZa4VBE1EYnwNuLq68tdffxk6DEEoM0RiFMjJyTF0CIJQqojE+IqbPHky0dHRfPnllzRv3pzVq1cTFRVF/fr12b59O++++y6enp6FPv7+t6apVCpZtWoVXbp0oU2bNowdO5bk5GStYvj3uubNm/PBBx9w6NChPOUqlYrZs2fTsmVLunfvzunTp9VlKSkp+Pj48M4779C+fXu+//579dj2/J8xf/582rZtS4sWLejRo4eYwFV4YSIxvuK+/fZbHB0dWblyJRcvXmT48OHqstDQUPbt28eaNWue+zkbN27k8OHDbNq0iRMnTmBlZcXs2bO1iqFatWps3ryZCxcu4OXlxeTJk4mLi1OXX7lyherVq3PmzBnGjBmDl5eXOul6e3tjZGTEwYMH2bVrF6dOnWL79u0F7nHy5EnOnz9PSEgIFy5cYPHixaVyrLNQNojE+BobPXo05ubm6rHMmmzdupXx48djb2+PiYkJXl5ehPyvvXsHaR2K4zj+tVYhouCLprhI0VmoL3xAxQ6CWdxEEaJUcBGkmyABwUHEqbh0ERR3oZsIDnUTNzsqCiqp+Ggn6yva3uXecBOf3e6l/89W8i8nOYQf57Tkn93dH23Dh4eHUVUVj8eDpmk0NzeTSqXs4/X19UxOTlJRUYGmaQQCAZLJJHd3d+zv77OwsEBVVRUNDQ1MTU19+Hy41+sll8txdnZGoVCgpaUFn89X3IQI8VtJN6otdX6//8e16XSa2dlZR+swj8dDJpNBVdUvv5tIJNjY2MA0TQAeHh4cLb1UVXV0zG5qauLm5oZ0Os3r66ujc0w+n/+wLVxvby8TExMsLS1hmiZDQ0PMz89TXV3942sU4g8JxhL2dxgpisLT05P9+e3tzfGKB7/fz/LyMh0dHUWNYZomhmGwublJMBikvLyckZERR8319TWFQsE+n6urK8LhsL06PTg4wOv9/lbVdR1d18lkMkSjUdbX14lGo0WdrxAgW+mS0NjYyOXl5Zc1gUCA5+dnkskklmURj8d5eXmxj4+PjxOLxexVXzabZW9v79uxHx8fKSsrs3tcbm9vc3Jy4qjJZrNsbW1hWRY7Ozucnp4yMDCAz+ejv7+flZUV7u/vyefzXFxccHh4+G6cVCrF0dERlmWhKAqVlZX/TWNc8e+RO6cEzMzMEI/H6ezs/PSPlpqaGhYXFzEMg1AohKIojq22ruuEw2EikQjBYJDR0VHH74SfaW1tJRKJMDY2Rl9fH8fHx7S3tztq2traOD8/p6enh1gsxtraGnV1dQCsrq5iWRaaptHV1cXc3By3t7fvxsnlchiGQXd3N4ODg9TW1jI9PV3MNAlhk36MQgjhIitGIYRwkWAUQggXCUYhhHCRYBRCCBcJRiGEcJFgFEIIFwlGIYRwkWAUQggXCUYhhHD5BTzsUM7ipCvrAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The accuracy is 0.4402332361516035\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Support Vector Machines\n",
        "\n",
        "SVM algorithm finds hyperplanes in n-dimensional space that optimally classifies the data points."
      ],
      "metadata": {
        "id": "MIj7Rmcde2h6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.svm import SVC\n",
        "from sklearn.model_selection import GridSearchCV"
      ],
      "metadata": {
        "id": "-8jcLtM9anPM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lsa = TruncatedSVD(10)\n",
        "svc = SVC(kernel='rbf',class_weight='balanced')"
      ],
      "metadata": {
        "id": "v9IbK3TyZkup"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = make_pipeline(lsa, svc)"
      ],
      "metadata": {
        "id": "h2sWotgzadNL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "param_grid = {'svc__C': [1,5,10,50], 'svc__gamma': [0.0001,0.0005,0.001,0.005]}\n",
        "grid = GridSearchCV(model, param_grid)\n",
        "grid.fit(X_train_tfidf, y_train)\n",
        "print(grid.best_params_)"
      ],
      "metadata": {
        "id": "BLO2XQzKcWlL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# BERT from Scratch / Encoder-only Attention Model\n",
        "\n",
        "See previous work on Transformers from scratch - similar idea but with encoder-only architecture based on the same multi-headed attention. Intellectual curiosity on the framework of the BERT model, practical usage still with HuggingFace versions of BERT and specialised variants. See next section."
      ],
      "metadata": {
        "id": "Z1ODakYZZnG5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CaAhsCwTMqZz"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "import re\n",
        "from random import *\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = re.sub(\"[.,!?\\\\-]\", '', text.lower()).split('\\n')\n",
        "word_list = list(set(\" \".join(sentences).split()))\n",
        "word_dict = {'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[MASK]': 3}\n",
        "\n",
        "\n",
        "for i, w in enumerate(word_list):\n",
        "    word_dict[w] = i + 4\n",
        "number_dict = {i: w for i, w in enumerate(word_dict)}\n",
        "vocab_size = len(word_dict)\n",
        "\n",
        "token_list = list()\n",
        "for sentence in sentences:\n",
        "    arr = [word_dict[s] for s in sentence.split()]\n",
        "    token_list.append(arr)"
      ],
      "metadata": {
        "id": "jDpK-QYuM3iC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "maxlen = 30 # maximum of length\n",
        "batch_size = 6\n",
        "max_pred = 5  # max tokens of prediction\n",
        "n_layers = 6 # number of Encoder of Encoder Layer\n",
        "n_heads = 12 # number of heads in Multi-Head Attention\n",
        "d_model = 768 # Embedding Size\n",
        "d_ff = 768 * 4  # 4*d_model, FeedForward dimension\n",
        "d_k = d_v = 64  # dimension of K(=Q), V\n",
        "n_segments = 2\n",
        "dropout_rate = 0.1"
      ],
      "metadata": {
        "id": "zSJQ-K3PM5xR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def make_batch():\n",
        "    batch = []\n",
        "    positive = negative = 0\n",
        "    while positive != batch_size/2 or negative != batch_size/2:\n",
        "        tokens_a_index, tokens_b_index= randrange(len(sentences)), randrange(len(sentences))\n",
        "        tokens_a, tokens_b= token_list[tokens_a_index], token_list[tokens_b_index]\n",
        "\n",
        "        input_ids = [word_dict['[CLS]']] + tokens_a + [word_dict['[SEP]']] + tokens_b + [word_dict['[SEP]']]\n",
        "\n",
        "        segment_ids = [0] * (1 + len(tokens_a) + 1) + [1] * (len(tokens_b) + 1)\n",
        "\n",
        "        #MASK LM\n",
        "        n_pred =  min(max_pred, max(1, int(round(len(input_ids) * 0.15)))) # 15 % of tokens in one sentence\n",
        "\n",
        "        cand_maked_pos = [i for i, token in enumerate(input_ids)\n",
        "                          if token != word_dict['[CLS]'] and token != word_dict['[SEP]']]\n",
        "        shuffle(cand_maked_pos)\n",
        "        masked_tokens, masked_pos = [], []\n",
        "        for pos in cand_maked_pos[:n_pred]:\n",
        "            masked_pos.append(pos)\n",
        "            masked_tokens.append(input_ids[pos])\n",
        "            if random() < 0.8:  # 80%\n",
        "                input_ids[pos] = word_dict['[MASK]'] # make mask\n",
        "            elif random() < 0.5:  # 10%\n",
        "                index = randint(0, vocab_size - 1) # random index in vocabulary\n",
        "                input_ids[pos] = word_dict[number_dict[index]] # replace\n",
        "\n",
        "        # Zero Paddings\n",
        "        n_pad = maxlen - len(input_ids)\n",
        "        input_ids.extend([0] * n_pad)\n",
        "        segment_ids.extend([0] * n_pad)\n",
        "\n",
        "    #     # Zero Padding (100% - 15%) tokens\n",
        "        if max_pred > n_pred:\n",
        "            n_pad = max_pred - n_pred\n",
        "            masked_tokens.extend([0] * n_pad)\n",
        "            masked_pos.extend([0] * n_pad)\n",
        "\n",
        "        if tokens_a_index + 1 == tokens_b_index and positive < batch_size/2:\n",
        "            batch.append([input_ids, segment_ids, masked_tokens, masked_pos, True]) # IsNext\n",
        "            positive += 1\n",
        "        elif tokens_a_index + 1 != tokens_b_index and negative < batch_size/2:\n",
        "            batch.append([input_ids, segment_ids, masked_tokens, masked_pos, False]) # NotNext\n",
        "            negative += 1\n",
        "\n",
        "    return batch"
      ],
      "metadata": {
        "id": "3BgSaf-sM7sp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_attn_pad_mask(seq_q, seq_k):\n",
        "    batch_size, len_q = seq_q.size()\n",
        "    batch_size, len_k = seq_k.size()\n",
        "    # eq(zero) is PAD token\n",
        "    pad_attn_mask = seq_k.data.eq(0).unsqueeze(1)  # batch_size x 1 x len_k(=len_q), one is masking\n",
        "    return pad_attn_mask.expand(batch_size, len_q, len_k)  # batch_size x len_q x len_k"
      ],
      "metadata": {
        "id": "jCeNn0NfM_cw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch = make_batch()"
      ],
      "metadata": {
        "id": "JqGZgCAhy0Nv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(tf.constant, zip(*batch))"
      ],
      "metadata": {
        "id": "5ThwN4Zny1Gh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids"
      ],
      "metadata": {
        "id": "9W61s2R21v62"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Embedding(tf.keras.layers.Layer):\n",
        "    def __init__(self):\n",
        "        super(Embedding, self).__init__()\n",
        "        self.tok_embed = tf.keras.layers.Embedding(vocab_size, d_model)  # token embedding\n",
        "        self.pos_embed = tf.keras.layers.Embedding(maxlen, d_model)  # position embedding\n",
        "        self.seg_embed = tf.keras.layers.Embedding(n_segments, d_model)  # segment(token type) embedding\n",
        "        self.norm = layers.LayerNormalization(d_model)\n",
        "\n",
        "    def forward(self, x, seg):\n",
        "        seq_len = x.size(1)\n",
        "        pos = tf.experimental.numpy.arange(seq_len)\n",
        "        pos = pos.unsqueeze(0).expand_as(x)  # (seq_len,) -> (batch_size, seq_len)\n",
        "        embedding = self.tok_embed(x) + self.pos_embed(pos) + self.seg_embed(seg)\n",
        "        return self.norm(embedding)"
      ],
      "metadata": {
        "id": "0lj_NGXaCSLw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def scaled_dot_product_attention(q,k,v,mask):\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention = matmul_qk / tf.math.sqrt(dk)\n",
        "  # add the mask\n",
        "  if mask is not None:\n",
        "    scaled_attention += (mask * -1e9)\n",
        "  \n",
        "  attention_weights = tf.nn.softmax(scaled_attention, axis=-1)\n",
        "  output = tf.matmul(attention_weights, v)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "metadata": {
        "id": "H87F-c7wNDdQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "\n",
        "    assert d_model % self.num_heads == 0\n",
        "\n",
        "    self.depth = d_model // self.num_heads\n",
        "    \n",
        "    self.wq = tf.keras.layers.Dense(d_model)\n",
        "    self.wk = tf.keras.layers.Dense(d_model)\n",
        "    self.wv = tf.keras.layers.Dense(d_model)\n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "\n",
        "  def split_heads(self, x, batch_size):\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(x, perm=[0,2,1,3])\n",
        "\n",
        "  def call(self, v, k, q, mask):\n",
        "    batch_size = tf.shape(q)[0]\n",
        "\n",
        "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
        "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
        "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
        "\n",
        "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
        "        q, k, v, mask)\n",
        "\n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "    concat_attention = tf.reshape(scaled_attention,\n",
        "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    return output, attention_weights"
      ],
      "metadata": {
        "id": "GbKdG3UkNi7Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),\n",
        "      tf.keras.layers.Dense(d_model)\n",
        "  ])"
      ],
      "metadata": {
        "id": "U32JH2Hb2OpS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "    \n",
        "    def __init__(self, d_model, FFN_units, n_heads, dropout_rate):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.FFN_units = FFN_units\n",
        "        self.n_heads = n_heads\n",
        "        self.d_model = d_model\n",
        "        self.dropout_rate = dropout_rate\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        # Build the multihead layer\n",
        "        self.multi_head_attention = MultiHeadAttention(self.d_model, self.n_heads)\n",
        "        self.ffn = point_wise_feed_forward_network(self.d_model, self.FFN_units)\n",
        "\n",
        "        self.norm_1 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.norm_2 = layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "        self.dropout_1 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.dropout_2 = layers.Dropout(rate=self.dropout_rate)\n",
        "        \n",
        "    def call(self, inputs, mask, training):\n",
        "        # Forward pass of the multi-head attention\n",
        "        attn_output, _ = self.multi_head_attention(inputs,\n",
        "                                              inputs,\n",
        "                                              inputs,\n",
        "                                              mask)\n",
        "        attn_output = self.dropout_1(attn_output, training=training)\n",
        "        attn_output = self.norm_1(attn_output + inputs)\n",
        "        outputs = self.ffn(attn_output)\n",
        "        outputs = self.dropout_2(outputs, training=training)\n",
        "        outputs = self.norm_2(outputs + attn_output)\n",
        "        \n",
        "        return outputs"
      ],
      "metadata": {
        "id": "MSOHW8YMNlO_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def torch_gather(x, indices, gather_axis):\n",
        "    # if pytorch gather indices are\n",
        "    # [[[0, 10, 20], [0, 10, 20], [0, 10, 20]],\n",
        "    #  [[0, 10, 20], [0, 10, 20], [0, 10, 20]]]\n",
        "    # tf nd_gather needs to be\n",
        "    # [[0,0,0], [0,0,10], [0,0,20], [0,1,0], [0,1,10], [0,1,20], [0,2,0], [0,2,10], [0,2,20],\n",
        "    #  [1,0,0], [1,0,10], [1,0,20], [1,1,0], [1,1,10], [1,1,20], [1,2,0], [1,2,10], [1,2,20]]\n",
        "\n",
        "    # create a tensor containing indices of each element\n",
        "    all_indices = tf.where(tf.fill(indices.shape, True))\n",
        "    gather_locations = tf.reshape(indices, [indices.shape.num_elements()])\n",
        "\n",
        "    # splice in our pytorch style index at the correct axis\n",
        "    gather_indices = []\n",
        "    for axis in range(len(indices.shape)):\n",
        "        if axis == gather_axis:\n",
        "            gather_indices.append(gather_locations)\n",
        "        else:\n",
        "            gather_indices.append(all_indices[:, axis])\n",
        "\n",
        "    gather_indices = tf.stack(gather_indices, axis=-1)\n",
        "    gathered = tf.gather_nd(x, gather_indices)\n",
        "    reshaped = tf.reshape(gathered, indices.shape)\n",
        "    return reshaped"
      ],
      "metadata": {
        "id": "q0ZR5PbP91w6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class BERT(tf.keras.layers.Layer):\n",
        "\n",
        "    def __init__(self,\n",
        "                 n_layers,\n",
        "                 FFN_units,\n",
        "                 n_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"BERT\"):\n",
        "      \n",
        "        super(BERT, self).__init__(name=name)\n",
        "        self.n_layers = n_layers\n",
        "        self.d_model = d_model\n",
        "        self.embedding = Embedding()\n",
        "        self.layers = [EncoderLayer(d_model, FFN_units,\n",
        "                                        n_heads,\n",
        "                                        dropout_rate) \n",
        "                           for _ in range(n_layers)]\n",
        "        \n",
        "        self.fc = tf.keras.layers.Dense(d_model, input_shape=(d_model,), activation=None)\n",
        "        self.linear = tf.keras.layers.Dense(d_model, input_shape=(d_model,), activation=None)\n",
        "        self.norm = layers.LayerNormalization(d_model)\n",
        "        self.classifier = tf.keras.layers.Dense(2, input_shape=(d_model,), activation=None)\n",
        "        \n",
        "        # decoder is shared with embedding layer\n",
        "        embed_weight = self.embedding.tok_embed\n",
        "        self.decoder = tf.keras.layers.Dense(d_model, input_shape=(vocab_size,), activation=None)\n",
        "        self.decoder.weight = embed_weight\n",
        "        # self.decoder_bias = nn.Parameter(torch.zeros(n_vocab))\n",
        "\n",
        "    def forward(self, input_ids, segment_ids, masked_pos, training):\n",
        "        output = self.embedding(input_ids, segment_ids)\n",
        "        enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids)\n",
        "\n",
        "        for layer in self.layers:\n",
        "            output, enc_self_attn = layer(output, enc_self_attn_mask, training)\n",
        "        # output : [batch_size, len, d_model], attn : [batch_size, n_heads, d_mode, d_model]\n",
        "  \n",
        "        h_pooled = tf.math.tanh(self.fc(output[:, 0])) # [batch_size, d_model]\n",
        "        logits_clsf = self.classifier(h_pooled) # [batch_size, 2]\n",
        "\n",
        "        masked_pos = masked_pos[:, :, None].expand(-1, -1, output.size(-1)) # [batch_size, max_pred, d_model]\n",
        "        # get masked position from final output of transformer.\n",
        "        \n",
        "        h_masked = torch_gather(output, 1, masked_pos) # masking position [batch_size, max_pred, d_model]\n",
        "        h_masked = self.norm(tf.keras.activations.gelu(self.linear(h_masked)))\n",
        "        logits_lm = self.decoder(h_masked)\n",
        "\n",
        "        return logits_lm, logits_clsf"
      ],
      "metadata": {
        "id": "lHuCNf5N1_Tt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "\n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "\n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "metadata": {
        "id": "khjQihLf-VDH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = BERT(    \n",
        "    n_layers = n_layers,\n",
        "    d_model = d_model,\n",
        "    n_heads = n_heads,\n",
        "    FFN_units = d_ff,\n",
        "    vocab_size = vocab_size, \n",
        "    dropout_rate = dropout_rate)\n",
        "\n",
        "learning_rate = CustomSchedule(d_model)\n",
        "criterion = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "                                    from_logits=True, reduction='none')\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,\n",
        "                                     epsilon=1e-9)\n",
        "\n",
        "batch = make_batch()\n",
        "input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(tf.constant, zip(*batch))\n",
        "\n",
        "for epoch in range(10):\n",
        "  with tf.GradientTape() as tape:\n",
        "\n",
        "    logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n",
        "    loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM\n",
        "    loss_lm = (loss_lm.float()).mean()\n",
        "    loss_clsf = criterion(logits_clsf, isNext) # for sentence classification\n",
        "    loss = loss_lm + loss_clsf\n",
        "\n",
        "  gradients = tape.gradient(loss, model.trainable_variables)\n",
        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "\n",
        "  if (epoch + 1) % 10 == 0:\n",
        "      print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
        "  loss.backward()\n",
        "  optimizer.step()"
      ],
      "metadata": {
        "id": "SJOwNsUy-Dcr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Normal HuggingFace Tokenizer and BERT\n",
        "\n",
        "Use of BertTokenizer with attention masks and special tokens and the BERT model. It’s a bidirectional transformer pretrained using a combination of masked language modeling objective and next sentence prediction on a large corpus comprising the Toronto Book Corpus and Wikipedia. Other points from HuggingFace website - \n",
        "\n",
        "BERT is a model with absolute position embeddings so it’s usually advised to pad the inputs on the right rather than the left.\n",
        "\n",
        "BERT was trained with the masked language modeling (MLM) and next sentence prediction (NSP) objectives. It is efficient at predicting masked tokens and at NLU in general, but is not optimal for text generation.\n",
        "\n",
        "Hence, BERT is a suitable model for this classification task."
      ],
      "metadata": {
        "id": "g0fi38hOzUlO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 5))\n",
        "sns.histplot([len(s) for s in X_train_preprocessed], bins=100)\n",
        "plt.title('Sentence Length')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "Pg56qbCxzTfX",
        "outputId": "5818f2be-835f-4330-cc72-513bd1705275"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAFBCAYAAAAok+gpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXBU1eHG8SebZAMYaEgIEJCREQsGqfISSbW+FMQqGGNri8ZUHKsWaNFSHRGKPxNEi4YwDr6AoIN26hvValVQiTqKb62OqNimQYtIK5IAmhCaEJLdzZ7fHzQLS26Szcvunt18PzOM5pzde88eLsmTe+45J8EYYwQAAAAruaLdAAAAALSNsAYAAGAxwhoAAIDFCGsAAAAWI6wBAABYjLAGAABgMcIaAMSxr7/+WmPGjJHP54t2UwB0EWENQMRs2bJFBQUFmjRpkiZPnqyCggL9/e9/7/Zxn3vuOV1xxRU90MKeNXXqVP31r3+N+3MCCK+kaDcAQO9QX1+vuXPnasmSJZo+fbq8Xq+2bNkit9sd7aYBgNW4swYgInbu3ClJysvLU2Jiovr06aOzzjpLJ598cuA1f/7znzV9+nSdfvrpuvbaa7V79+5A3ZgxY/TUU0/pRz/6kXJycnT77bfLGKMdO3aouLhYW7du1YQJE5STkyNJ8ng8Kikp0Q9/+EOdeeaZKioqUmNjoyTpgw8+0DnnnKNHHnlEZ5xxhs466yw9++yzgXM1Njbq7rvv1pQpUzRp0iRdccUVgfdu3bpVBQUFysnJUX5+vj744INO94Xf79dDDz2kadOmKTc3V/Pnz1dtba2kI8OWf/nLX/TDH/5Qubm5evDBB4PatnDhQp1++umaPn26Hn74YZ1zzjmSpAULFqiyslJz587VhAkT9PDDDwfet2HDBsfjAYgBBgAioK6uzkyePNnccsstZvPmzaa2tjao/rXXXjPTpk0zX3zxhfF6vWbVqlXm8ssvD9SPHj3azJ492xw4cMDs3r3b5ObmmrfeessYY8yzzz5rCgoKgo73+9//3syZM8fs37/f1NXVmTlz5pgVK1YYY4x5//33TXZ2tlm5cqXxeDxm8+bN5tRTTw20acmSJebKK680e/bsMT6fz3z00UemqanJ7Nmzx0yePNls3rzZNDc3m3fffddMnjzZVFdXO37mKVOmmPfee69V+R/+8Aczc+ZMU1VVZZqamsxtt91mbrzxRmOMMbt27TKjR482t956qzl06JDZtm2bOeWUU8wXX3xhjDGmtLTU/PznPze1tbWmqqrK5OXlmbPPPrvNc3Z0PAD2484agIhITU3Vk08+qYSEBN12220644wzNHfuXH377beSpPXr12v27NkaNWqUkpKSNHfuXG3bti3o7tovf/lLDRgwQMOGDVNubq4+++wzx3MZY/T0009r8eLFSktLU2pqqubMmaOXXnop8JqkpCTNmzdPycnJOvfcc9WvXz/t3LlTfr9fzz77rG699VYNGTJEiYmJmjhxotxut1544QWdc845Ovfcc+VyufSDH/xA48aN01tvvdWpvli/fr1uvPFGDR06VG63W9dff73KysqCJgFcf/316tOnj04++WSdfPLJgc/6yiuvaM6cOfrOd76joUOH6qqrrgrpnG0dD4D9eGYNQMSMGjVKd999tyRpx44dWrBggZYtW6Z77rlHlZWVWrZsmUpKSgKvN8Zo7969Gj58uCQpMzMzUNe3b18dPHjQ8Tw1NTU6dOiQLr300qBj+f3+wNdpaWlKSjryLbBv375qaGjQ/v371dTUpBEjRrQ6bmVlpTZt2qQ333wzUObz+ZSbm9upfqisrNS8efPkch35fdnlcqm6ujrw9aBBg1q1TZL27dunrKysQN3QoUNDOmdbxwNgP8IagKgYNWqULr30Uv3pT3+SJGVlZWnu3LnKz8/v9LESEhKCvh44cKD69Omjl156SUOGDOnUsQYOHKiUlBTt2rUr6Hm6ljZecskluvPOOzvdxqMNHTpUy5Yt06RJk1rVff311+2+NzMzU3v27NFJJ50kSdqzZ0+32gLAfgyDAoiIHTt26JFHHgmEi6qqKm3cuFGnnXaaJKmgoEAPPfSQtm/fLkmqq6vTK6+8EtKxMzIytHfvXnk8HkmH71LNnDlTy5YtC9yt2rt3r955550Oj+VyufTTn/5Ud911l/bu3avm5mZ98skn8ng8ys/P15tvvql33nlHzc3Nampq0gcffNBuYPJ6vWpqagr88fl8uuKKK7Ry5crAEG9NTY1ef/31kD7r9OnTtXbtWh04cEB79+7V448/HlQ/aNAg7dq1K6RjAYgNhDUAEZGamqpPP/1UM2fO1Pjx43XZZZdp9OjRWrRokSTp/PPP13XXXaebbrpJEydOVF5ent5+++2Qjv39739fJ510ks4666zAkOSCBQt0wgkn6LLLLtPEiRN19dVXB2akdmThwoUaPXq0fvazn2ny5MlasWKF/H6/srKytHr1aq1du1ZnnHGGzj33XK1bty5oePVYs2fP1qmnnhr4c//99+uqq67S1KlTdc0112jChAm67LLLQl5vbt68eRo6dKjOO+88XX311brggguClj+ZPXu2HnzwQeXk5GjdunUhHROA3RKMMSbajQAAdM2TTz6pl19+udUdNgDxgztrABBD9u3bp48++kh+v19ffvmlHn30UU2bNi3azQIQRkwwAIAY4vV6VVxcrK+//lr9+/fXRRddpMLCwmg3C0AYMQwKAABgMYZBAQAALEZYAwAAsBhhDQAAwGJxP8Fg//6D8vvD91heRkaqqqvrw3Z8HEFfRwb9HDn0dWTQz5FDX3edy5WggQOPc6yL+7Dm95uwhrWWcyAy6OvIoJ8jh76ODPo5cujrnscwKAAAgMUIawAAABYjrAEAAFiMsAYAAGAxwhoAAIDFCGsAAAAWI6wBAABYjLAGAABgMcIaAACAxeJ+BwMAiBcev1FdgzeorH+/ZLldCVFqEYBIIKwBQIyoa/Bq3YvlQWXX5o9TRqo7Si0CEAkMgwIAAFiMsAYAAGAxwhoAAIDFeGYNABCEiQyAXQhrABBnQg1bTq+TJL8xenTDP4PKmMgARA9hDQDiTKizRp1eJ0mzZowNa/sAdA5hDQDQoaQkl6rrPUFl/fokqaHRF1TGcCnQ8whrAIAONTT69NjLFUFls2aMbVXGcCnQ85gNCgAAYDHurAFADHManvQbE6XWAAgHwhoAxLC2hicBxA+GQQEAACxGWAMAALAYYQ0AAMBihDUAAACLEdYAAAAsFrGw9uabb+rHP/6xLrnkEuXn5+vVV1+VJO3cuVOXX365LrjgAl1++eX697//HXhPe3UAAAC9QUTCmjFGt9xyi5YvX64XXnhBy5cv18KFC+X3+1VcXKzCwkKVlZWpsLBQRUVFgfe1VwcAANAbROzOmsvlUl1dnSSprq5OgwcP1v79+1VRUaG8vDxJUl5enioqKlRTU6Pq6uo26wAAAHqLiCyKm5CQoJUrV+rXv/61+vXrp4MHD+qhhx5SVVWVhgwZosTERElSYmKiBg8erKqqKhlj2qxLT08P+dwZGalh+UxHy8zsH/Zz4DD6OjLo58hpq69rDjSqtr4pqCzBlaDk5OBv264Qy9zupFbnqvMcaPW6zhwz1PPYwMY2xSv6uudFJKz5fD6tXbtWq1ev1qRJk/TRRx/pt7/9rZYvXx72c1dX18vvD9/WK5mZ/fXNN3VhOz6OoK8jg36OnPb6urreo3UvlgeVzZoxVl6vL6jM7zchlXk8vlbn8nh8rV7XmWOGep5o45qOHPq661yuhDZvMEUkrG3btk379u3TpEmTJEmTJk1S3759lZKSor1796q5uVmJiYlqbm7Wvn37lJWVJWNMm3UAgNjh8RvVNXiDyvr3S5bblRClFgGxJSLPrA0dOlR79uzRl19+KUnasWOHqqurdcIJJyg7O1sbN26UJG3cuFHZ2dlKT09XRkZGm3UAEKs8fqPqeo+q6z36cvcBVdd75Anj3X8b1DV4te7F8qA/x4Y3AG2LyJ21zMxMLVmyRPPnz1dCwuHfpJYtW6a0tDQtWbJEixYt0urVqzVgwACVlJQE3tdeHQDEopbgIknJyUnyen26Nn+cMlLdUW4ZAFtFJKxJUn5+vvLz81uVjxo1Ss8884zje9qrAwAA6A3YwQAAAMBihDUAAACLEdYAAAAsRlgDAACwGGENAADAYhGbDQoAvY3TYrB+E99rqgHoeYQ1AOikUFfkP3pNtRazZowNe/sAxBfCGgB0klMIY2FbAOHCM2sAAAAW484aACDikpJcqq73BJWxuTvgjLAGAIi4hkafHnu5IqiMoWTAGcOgAAAAFiOsAQAAWIywBgAAYDGeWQOA/wl1/TQAiCTCGgD8D+unAbARw6AAAAAWI6wBAABYjLAGAABgMZ5ZA9DrOE0kkCS/MVFoDQC0j7AGoNdxmkggSbNmjO3yMZ22TyL8AegJhDUA6AFO2yd1J/wBQAueWQMAALAYYQ0AAMBiDIMCAKzg9NwfO0gAhDUAgCWcnvtjBwmAYVAAAACrEdYAAAAsRlgDAACwGGENAADAYoQ1AAAAizEbFABgLZbzAAhrAKLIaUN1fhDjaCznARDWAESR04bq/CAGgGCENQBW6c6wF3fqei/+7hHPCGsArNKdYS/u1PVe/N0jnjEbFAAAwGKENQAAAIsR1gAAACxGWAMAALAYEwwARITTbD2/MSG9l4VRAfRmhDUAEeE0W2/WjLEhvZeFUQH0ZgyDAgAAWIywBgAAYDHCGgAAgMV4Zg1AyNjSBwAij7AGIGRs6QMAkccwKAAAgMW4swYgrjmt0Rbq+m4AYAPCGoC45rRGW6jruwGADSIW1pqamrRs2TL97W9/U0pKisaPH6877rhDO3fu1KJFi1RbW6u0tDSVlJRo5MiRktRuHYDw6s6OAwCAnhOxsFZaWqqUlBSVlZUpISFB3377rSSpuLhYhYWFuuSSS/TCCy+oqKhIf/zjHzusAxBe3dlxAADQcyIyweDgwYN6/vnnNX/+fCUkHJ7iP2jQIFVXV6uiokJ5eXmSpLy8PFVUVKimpqbdOgAAgN4iInfWdu3apbS0ND3wwAP64IMPdNxxx2n+/Pnq06ePhgwZosTERElSYmKiBg8erKqqKhlj2qxLT0+PRLMBAACiLiJhrbm5Wbt27dLYsWO1cOFCffrpp5o7d67uvffesJ87IyM17OfIzOwf9nPgMPo6MjIz+6vOc0DJycHfIlyuhFZlbndSSH8voR4v1DKn83bnHJE6z7FlyclJYT9HZz5HpM4TiXMc/Tq+d0QOfd3zIhLWsrKylJSUFBjSPO200zRw4ED16dNHe/fuVXNzsxITE9Xc3Kx9+/YpKytLxpg26zqjurpefn/4HorOzOyvb76pC9vxcQR9HRkt/ezx+OT1+oLq/H7jUObXZzurg8qcdjUI/XihlXk8vlbXQ3fOEanzHF2WnJwkr9cX1nN09nNE6jyROEfL6/jeETn0dde5XAlt3mCKyDNr6enpys3N1XvvvSfp8CzP6upqjRw5UtnZ2dq4caMkaePGjcrOzlZ6eroyMjLarANgj4ZGn9a9WB7059hZpACArovYbNDbb79dixcvVklJiZKSkrR8+XINGDBAS5Ys0aJFi7R69WoNGDBAJSUlgfe0VwfgCPbsBID4FbGwNmLECD322GOtykeNGqVnnnnG8T3t1QE4gj07ASB+sTcoAACAxQhrAAAAFiOsAQAAWIyN3AEAcSkpyaXqeo/qPAfk8Rxe1oOJN4hFhDUAQFxqaPTpsZcrAuvZSdKcS09Vnc8f9DoCHGxHWAMQtPRHy10IvwnfYtJAtLQEuKMxcxq2I6wBCFr6o+UuxKwZY6PcKvSkliHBoxHIgdhAWAOAXsDpjhKBHIgNzAYFAACwGGENAADAYgyDAjHGaR9Qnj0CgPhFWANijNM+oDx7BADxi2FQAAAAi3FnDQCiLJ6W1YinzwLYgrAGAFEWT8tqxNNnAWzBMCgAAIDFuLMGAOjVnIZu2S8UNiGsAXHK6QeQxA8h4FjsFwrbEdaAOOX0A0jihxAAxBqeWQMAALAYYQ0AAMBihDUAAACLhRzWXnnlFcfyTZs29VhjAAAAECzksHbrrbc6lhcVFfVYYwAAABCsw9mgu3btkiQZYwL/f3Sd282sMgAAgHDpMKydf/75SkhIkDFG559/flDdoEGDdMMNN4StcQAAAL1dh2Hts88+kyRdeeWVevzxx8PeIAAAABwR8jNrBDUAAIDIC3kHg127dmnlypXatm2bGhoaguo2b97c0+0CIMnjN6pr8AaV+Y2JUmt6J6dtu/g7ABBJIYe1m2++WSNGjNDChQvVt2/fcLYJwP/UNXi17sXyoLJZM8ZGqTW9k9O2XfwdAIikkMPa9u3b9dRTT8nlYh1dAACASAk5eZ1++umqqGi9KTQAAADCJ+Q7a8OHD9d1112n888/X4MGDQqqmz9/fo83DAAAAJ0Ia4cOHdKUKVPk8/m0Z8+ecLYJAAAA/xNyWLvrrrvC2Q4gbjnN6OzfL1luV0KUWgQAiCWdWrqjLSNGjOiRxgDxyGlG57X545SRylZtAICOhRzWjt52qkVCwuE7A9u2bev5lgEAACD0sNay7VSLb775Rg888IBycnJ6vFFAvHNaaJWhUQCAk5DD2rEyMzN166236oILLtDFF1/ck20C4p7TQqsMjQIAnHRrhdsvv/xShw4d6qm2AAAA4Bgh31krLCwMPKMmHV7K44svvtC8efPC0jAAAAB0IqzNnDkz6Ou+ffvq5JNP1siRI3u6TQAAAPifkMPaT37yk3C2AwAAAA5CfmbN6/Xqvvvu03nnnafvfe97Ou+883TffffJ4/F0/GYAAAB0Sch31kpLS/X3v/9dt99+u4YNG6bKykqtXr1a9fX1Wrx4cTjbCABAgNPSN/6j1gAF4k3IYW3Tpk164YUXNHDgQEnSiSeeqLFjx+qSSy4hrAEAIsZp6ZtZM8ZGqTVA+IU8DGra+K2lrXIAAAB0X8h31i688EL96le/0rx58zRs2DDt3r1bDz74oC688MJwtg+IKU6btjM8Ex4MhQHoLUIOawsWLNCDDz6opUuXat++fRoyZIguuugi/epXvwpn+4CY4rRpe28cnolEkGIoDEBv0WFY++ijj/TGG29owYIFmj9/vubPnx+oKy0tVUVFhcaPHx/WRgKILQQpxCunu+fs64tw6zCsrV27VoWFhY51ubm5WrNmjdasWdPjDQMAwDZOd8/Z1xfh1uEEg23btunss892rDvzzDNVXl7uWNeWBx54QGPGjNG//vUvSdLWrVuVn5+vCy64QNdcc42qq6sDr22vDgAAoDfoMKzV19fL6/U61vl8Ph08eDDkk/3zn//U1q1bNXz4cEmS3+/XggULVFRUpLKyMuXk5GjFihUd1gEAAPQWHYa1E088Ue+++65j3bvvvqsTTzwxpBN5PB4tXbpUS5YsCZSVl5crJSVFOTk5kqSCggJt2rSpwzoAAIDeosNn1q6++moVFxfL7/dr2rRpcrlc8vv9ev3117V06VItWrQopBPde++9ys/P1/HHHx8oq6qq0rBhwwJfp6eny+/3q7a2tt26tLS0kD9gRkZqyK/tqszM/mE/Bw6zva/rPAeUnBz8z8rlSgipzO1OavX5unM8p7JQz5OcnNSt88RiWbTObVtfR/Pc4Sxr+W93/j1Kzv8m23ptb0Vf9LwOw9rFF1+sb7/9VgsXLpTX61VaWppqa2uVnJys3/zmN8rLy+vwJJ988onKy8t1880390ijO6O6ul5+f/jWXsrM7K9vvqkL2/FxRCz0tcfjk9frCyrz+01IZR6Pr9Xn687xnMpCOU9ycpK8Xl+3zhOLZdE4t419Hc1zh6uspZ87816nfyct5aG+tjeKhe/TtnK5Etq8wRTSOmu/+MUvNHPmTH3yySeBO1sTJkxQampod60+/PBD7dixQ+edd54kac+ePbr22ms1a9YsVVZWBl5XU1Mjl8ultLQ0ZWVltVkH2IAFcAEAkRDyoripqaltzgrtyOzZszV79uzA11OnTtWaNWt00kkn6emnn9aWLVuUk5Oj9evXB3ZEGDdunBobGx3rABuwAC4AIBJCDmvh4HK5tHz5chUXF6upqUnDhw9XaWlph3UAAAC9RVTC2htvvBH4/4kTJ2rDhg2Or2uvDgAAoDeI6p01AJHHBugAEFsIa8Ax4n3vP/btBIDYQlgDjsHefwAAm3S4gwEAAACih7AGAABgMcIaAACAxXhmDQCAbnCaYR1Pk5IQfYQ1AAC6wWmGNZOS0JMYBgUAALAYYQ0AAMBiDIOiV3NaAJfV/AEANiGsoVdzWgCX1fwBADZhGBQAAMBihDUAAACLEdYAAAAsxjNrAAAcw2mhW4kJSIgOwhoAAMdwWuhWYgISooOwBljC6Td5fosHABDWAEs4/SbPb/EAACYYAAAAWIywBgAAYDHCGgAAgMUIawAAABYjrAEAAFiM2aAAAPQwp6V4+vdLltuVEKUWIZYR1tBrePxGdQ3eoDLWMQMQDk5L8cy59FTV+fxBZQQ4hIKwhl6jrsGrdS+WB5WxjhmASHEKcNfmj1NGqjtKLUKs4Jk1AAAAi3FnDQgBW0EBAKKFsAaEgK2gAADRwjAoAACAxbizhrjEzE8AQLwgrCEuMfMTABAvGAYFAACwGGENAADAYoQ1AAAAixHWAAAALEZYAwAAsBhhDQAAwGKENQAAAIsR1gAAACxGWAMAALAYYQ0AAMBihDUAAACLEdYAAAAsxkbuAABYxOM3qmvwBpX175cstyshSi1CtBHWAACwSF2DV+teLA8quzZ/nDJS3VFqEaKNYVAAAACLEdYAAAAsxjAorOT0zEaiuzFKrQEAIHoiEtb279+vW265RV999ZXcbrdOOOEELV26VOnp6dq6dauKiorU1NSk4cOHq7S0VBkZGZLUbh3im9MzG7+eOV5NTb6gMh66BQDEu4gMgyYkJOi6665TWVmZNmzYoBEjRmjFihXy+/1asGCBioqKVFZWppycHK1YsUKS2q1D73Tw0OEAd/SfY+++AQAQbyIS1tLS0pSbmxv4evz48aqsrFR5eblSUlKUk5MjSSooKNCmTZskqd06AACA3iLiEwz8fr+eeuopTZ06VVVVVRo2bFigLj09XX6/X7W1te3WAQAA9BYRn2Bwxx13qF+/frryyiv12muvhf18GRmpYT9HZmb/sJ+jt6nzHFBycuvL89gytzvJsf+d3u9yJfT6slBfm5ycZFW7beqbni6zra+jee5wlrX817a+cfoe5vT9KyUlSXUef1BZWmqK0r/Tp1Ubo42fiT0vomGtpKRE//nPf7RmzRq5XC5lZWWpsrIyUF9TUyOXy6W0tLR26zqjurpefr/psc9wrMzM/vrmm7qwHb+38nh88np9rcqPLfN4fI797/R+v9/0+rJQXpucnCSv12dVu23pm54us7Gvo3nucJW19HN3jxeONjp9D3P6/lV30KPHXq4IKrs2f5yaPXY9t8vPxK5zuRLavMEUsWHQe+65R+Xl5Vq1apXc7sOrMI8bN06NjY3asmWLJGn9+vW68MILO6wDAADoLSJyZ2379u1au3atRo4cqYKCAknS8ccfr1WrVmn58uUqLi4OWp5DklwuV5t1AAAAvUVEwtp3v/tdff755451EydO1IYNGzpdBwAA0Buw3RQAAIDFCGsAAAAWI6wBAABYjLAGAABgsYgvigsAALovKcml6npPUFn/fslyuxKi1CKEC2ENAIAocQpcfhPaQu4NjT7HhXIzUt091j7YgbCGqPP4jeoaglfhDvWbFQDEMqfANWvG2Ci1BrYirCHq6hq8WvdieVAZ36wAADiMsIaI4i4aAACdQ1hDRPX0XTSn5z0kAiAAIH4Q1hDTnJ73kBhGBQDED9ZZAwAAsBhhDQAAwGKENQAAAIsR1gAAACxGWAMAALAYs0EBAIgT7BcanwhrAADECfYLjU8MgwIAAFiMsAYAAGAxwhoAAIDFCGsAAAAWY4IBeoTHb1TX4A0qYwYSAADdR1hDj6hr8Grdi+VBZcxAAgCg+whrCBun9X78xkSpNQAAxCbCGsLGab2fWTPGRqk1AADEJiYYAAAAWIw7a2gXEwcAIP7wvT22ENbQLqeJA3MuPVV1Pn9QGc+iAUDsYFJYbCGsodN4Fg0AgMghrAEAAMcZ/AyN2oGwhgCnZxgY3gSA2BbqMkpOoyYMjdqBsIYAp2cYGN4EgNjGoyuxj6U7AAAALEZYAwAAsBhhDQAAwGKENQAAAIsR1gAAACxGWAMAALAYS3cAAABHLJRrB8IaAABw5LRGm9P+0NLhEIfwIKzFGaddCPr1SVJDoy+ojN+MAABd4RTgpMO7HSA8CGtxpq1dCNhCBACA2ERYi2Hs5QkAQPwjrMWw7uzlGerGvgAAILoIa70UG/sCABAbWGcNAADAYtxZiyKnZ86cZmk6vU5i2BIAYI+kJJe+3H1AHs+R1QdYjaBnENaiyOmZM6dZmk6vkxi2BADYo6HRp/Wv/Ute75FwxmoEPYOwBgAAIoZdETqPsAYAACIm1F0RCHBHWB/Wdu7cqUWLFqm2tlZpaWkqKSnRyJEjo90sAADQQ5wCHMOlR1g/G7S4uFiFhYUqKytTYWGhioqKot2kLvH4jarrPUF/nCYItNwe7uh1AACgd7D6zlp1dbUqKir06KOPSpLy8vJ0xx13qKamRunp6SEdwxWBW6ihnKOx0afn3tweVPaTKd/VwP4pQWVen19/CeF1kpSc5GpVTpl97bGpLJTXJiUnyedNtKrdtvRNT5fZ2NfRPHe4ylr6ubvHs+Gz2FrWUp7WPyXQ1909Zp+UJP33mJmkfVIS1djU3KWy4/okKdniYdX2skSCMfbetikvL9fChQv10ksvBcpmzJih0tJSnXLKKVFsGQAAQGRYPwwKAADQm1kd1rKysrR37141Nx++ldnc3Kx9+/YpKysryi0DAACIDKvDWkZGhrKzs7Vx40ZJ0saNG5WdnR3y82oAAACxzupn1iRpx44dWrRokf773/9qwIABKikp0YknnhjtZnX5fk8AAASFSURBVAEAAESE9WENAACgN7N6GBQAAKC3I6wBAABYjLAGAABgMcIaAACAxazebspmbDAfXlOnTpXb7VZKyuHtR26++WadffbZ2rp1q4qKitTU1KThw4ertLRUGRkZUW5t7CgpKVFZWZl2796tDRs2aPTo0ZLav5651rumrb5u69qWxPXdBfv379ctt9yir776Sm63WyeccIKWLl2q9PT0dvuTvu6c9vp5zJgxGj16tFyuw/d/li9frjFjxkiS3njjDS1fvlzNzc065ZRTdNddd6lv377R/CixyaBLZs2aZZ5//nljjDHPP/+8mTVrVpRbFF+mTJliPv/886Cy5uZmM23aNPPhhx8aY4xZtWqVWbRoUTSaF7M+/PBDU1lZ2ap/27ueuda7pq2+drq2jeH67qr9+/eb999/P/D13XffbX73u9+125/0dee11c/GGDN69GhTX1/f6j319fXmzDPPNDt37jTGGLN48WJz//33R6S98YZh0C5o2WA+Ly9P0uEN5isqKlRTUxPllsW38vJypaSkKCcnR5JUUFCgTZs2RblVsSUnJ6fVDiDtXc9c613n1Nft4frumrS0NOXm5ga+Hj9+vCorK9vtT/q689rq5/a8/fbbGjduXOBOfEFBgV555ZVwNjNuMQzaBVVVVRoyZIgSExMlSYmJiRo8eLCqqqrYXaEH3XzzzTLGaNKkSbrppptUVVWlYcOGBerT09Pl9/sDw3PomvauZ2MM13oYHHttDxgwgOu7B/j9fj311FOaOnVqu/1JX3fP0f3cYtasWWpubtY555yjG264QW63u1U/Dxs2TFVVVdFocszjzhqs9MQTT+jFF1/Us88+K2OMli5dGu0mAT2Cazt87rjjDvXr109XXnlltJsS147t582bN+u5557TE088oS+++EKrVq2KcgvjD2GtC9hgPvxa+tLtdquwsFAff/yxsrKygm6719TUyOVy8ZtwN7V3PXOt9zyna7ulnOu760pKSvSf//xHK1eulMvlarc/6euuO7afpSPXdGpqqmbOnNnmNV1ZWcn3ji4irHUBG8yHV0NDg+rq6iRJxhi9/PLLys7O1rhx49TY2KgtW7ZIktavX68LL7wwmk2NC+1dz1zrPauta1sS13c33HPPPSovL9eqVavkdrsltd+f9HXXOPXzgQMH1NjYKEny+XwqKysLXNNnn322/vGPf+jf//63pMP9PH369Ki0PdaxN2gXscF8+OzatUs33HCDmpub5ff7NWrUKP3f//2fBg8erI8//ljFxcVB0+0HDRoU7SbHjDvvvFOvvvqqvv32Ww0cOFBpaWl66aWX2r2euda7xqmv16xZ0+a1LYnruwu2b9+uvLw8jRw5Un369JEkHX/88Vq1alW7/Ulfd05b/XzdddepqKhICQkJ8vl8mjBhghYvXqzjjjtOkvT666+rtLRUfr9f2dnZuvvuu9WvX79ofpSYRFgDAACwGMOgAAAAFiOsAQAAWIywBgAAYDHCGgAAgMUIawAAABYjrAEAAFiMsAYAAGAxwhoAAIDF/h9HZl9PSBD6EQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',\n",
        "                                         do_lower_case = True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "52c890e085194779ad441ee9d8cad044",
            "3d4672919c4d47b99b7c860702d13a5e",
            "4a468ba3c8de40fd96742f7da7155707",
            "b7f5cec427d54f4f8fafd7e32d6749ff",
            "e078ba07b2594beeb8ead3a8bb332d3d",
            "2dd4c3d747644d5fac0bb3a274c407ad",
            "906054da4a904253873f86301553e951",
            "cbb74f72e1134677ba5722dbbde78cf6",
            "0757c0f912b14ad9a13b0dd547887d50",
            "472672951d964d7b8bc0043177f639cc",
            "2a784bcf90b04d3db6948916fa47306c",
            "57b61a07d2d64b81b478a279b0fc6383",
            "f4421563eae94c088dd37de667c7316c",
            "7aaee56048fd445684b9dec8b7a6411e",
            "40b6af3f37e54af0abb721b5bd11ad27",
            "25aba31f64d348b1afb66a9b1cd4abe9",
            "e0b33af1afb94720a863f90997ef7672",
            "fcbed9b3cb3b4bf399dc635a077967e7",
            "5268e399a153493fb07ae428d1da1d81",
            "3623f23a041d437ba047180173f0cdb0",
            "44ed1e2ac3674f6db7647b635ff4ec8f",
            "62bb96c2e9c14cc383b59ae0eb7824ce",
            "082cfc4e8a534426934a19ed9b352ecc",
            "b76851e764a644ea8fdb975f572b9c62",
            "ef7ea7c617524d519a2db15c235cae0b",
            "4c565c7fe1a243efb2ba99f7751f2153",
            "2836257412f946cab8e6efe7a135a894",
            "e8b3be37a073438c92add3e119f7a5e1",
            "95edc75523b84e87b86b693422aeba2f",
            "a4bc63bb9c9140c6bf9e66df3653d12f",
            "237045fd10f5414193ffa9416567a419",
            "df8ca0e535764192bc39c4732c3298c5",
            "a0c08336037c4d9ba6f1b67972f78056",
            "58456a7e6fc84fbc958c1245aceb1784",
            "d829504a53f84566a9f1174c0880a8d1",
            "d2ce9ee1b04448aba3fbea142cefdcb6",
            "246e51b1554f4952a88ae05b318b5a51",
            "cdcc0e95db6a4b1e842341f9a0c6af43",
            "8d22f3640ee7414f92309888ec419644",
            "b4a006f1065748e4a08bd6c544963a48",
            "07719dbf516a47319c4172543df96341",
            "fd40d0dad31840b5aaaf60bbd3821138",
            "d8abd2148e6648da989358eb92c2ef4c",
            "c6817c01178044528d8b035415b8f8a3"
          ]
        },
        "id": "Q7LntW4UzmUy",
        "outputId": "02d9bd8e-421f-4a16-fbe3-9ea32b20d9d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "52c890e085194779ad441ee9d8cad044",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "57b61a07d2d64b81b478a279b0fc6383",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "082cfc4e8a534426934a19ed9b352ecc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "58456a7e6fc84fbc958c1245aceb1784",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Add_special_tokens: adds the special [CLS] and [SEP] tokens to the beginning and the end of the sentences\n",
        "\n",
        "Return_attention_mask: creates attention mask by creating array of 0s (pad token) and 1s (real token)\n",
        "\n",
        "Pad_to_max_length: creates constant length for all sequences by padding them to max length"
      ],
      "metadata": {
        "id": "zFPrmzkK4VkO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#encode train set\n",
        "encoded_data_train = tokenizer.batch_encode_plus(X_train_preprocessed,\n",
        "                                                add_special_tokens = True,\n",
        "                                                return_attention_mask = True,\n",
        "                                                padding = 'max_length',\n",
        "                                                max_length = 256,\n",
        "                                                truncation = True,\n",
        "                                                return_tensors = 'pt')\n",
        "                                                \n",
        "#encode validation set\n",
        "encoded_data_val = tokenizer.batch_encode_plus(X_val_preprocessed,\n",
        "                                                add_special_tokens = True,\n",
        "                                                return_attention_mask = True,\n",
        "                                                padding = 'max_length',\n",
        "                                                max_length = 256,\n",
        "                                                truncation = True,\n",
        "                                                return_tensors = 'pt')"
      ],
      "metadata": {
        "id": "IyNlH7NM3XIE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_data_train['input_ids']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0lJj3TPY4gS5",
        "outputId": "b22f9010-98ac-42a6-c35c-6b08a5d3811e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  101, 13025,  5324,  ...,     0,     0,     0],\n",
              "        [  101,  3352,  6739,  ...,     0,     0,     0],\n",
              "        [  101,  2619,  2557,  ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [  101, 10321, 15358,  ...,     0,     0,     0],\n",
              "        [  101,  7325,  9597,  ...,     0,     0,     0],\n",
              "        [  101,  2092,  2056,  ...,     0,     0,     0]])"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_data_train['attention_mask']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xMtFc5T4u7I",
        "outputId": "49a67a95-a264-4357-8f2a-f4a41b21b75b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch"
      ],
      "metadata": {
        "id": "ddg_jMfS44J-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train set\n",
        "input_ids_train = encoded_data_train['input_ids']\n",
        "attention_masks_train = encoded_data_train['attention_mask']\n",
        "labels_train = torch.tensor(y_train)\n",
        "\n",
        "#validation set\n",
        "input_ids_val = encoded_data_val['input_ids']\n",
        "attention_masks_val = encoded_data_val['attention_mask']\n",
        "labels_val = torch.tensor(y_val)"
      ],
      "metadata": {
        "id": "XFwZm2PV4xWo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labels_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QayEJamE5Jyo",
        "outputId": "c49f1b7b-767d-4efc-99c9-b426f6f97500"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3, 2, 4,  ..., 1, 2, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Since BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers, the pre-trained BERT model can be fine-tuned with just one additional output layer (Devlin et al., 2019)."
      ],
      "metadata": {
        "id": "m--KV8G-5SyH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertForSequenceClassification\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', \n",
        "                                                      num_labels = 5, \n",
        "                                                      output_attentions = False, \n",
        "                                                      output_hidden_states = False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156,
          "referenced_widgets": [
            "516675a4dd5e4d2da79ec6324dde098e",
            "736cecabae8c4b63b8f149dc24b3b489",
            "e84b55ddfff744fe9931711f966612af",
            "d6b05dc8571a4f98a2963ba97ef86652",
            "45f7b401b08042f181b065d51684eab2",
            "56cf6baa9bc644b2a0aacb033d1d4c18",
            "6a3c673c3ad1460ab3732e3ddcdefd4a",
            "1c6ddcd1b8504447843d445c004b95f8",
            "29a4b3d965bc4415881f9733f087c669",
            "3ad22f34224747e3a7d83f94dc070302",
            "0864242456d54576b0370103475c5cc8"
          ]
        },
        "id": "gIhYzF1n5L-E",
        "outputId": "a914cf4b-7f3f-46b1-acc2-4310fb04575a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "516675a4dd5e4d2da79ec6324dde098e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import TensorDataset"
      ],
      "metadata": {
        "id": "zL_BFLdg6OHj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "TensorDataset creates DataLoader which provides a way to create a dataset out of the data that is already loaded into memory. It accepts data in numpy array and tensor form."
      ],
      "metadata": {
        "id": "veCbSIB36XOJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#train set\n",
        "dataset_train = TensorDataset(input_ids_train, \n",
        "                              attention_masks_train,\n",
        "                              labels_train)\n",
        "\n",
        "#validation set\n",
        "dataset_val = TensorDataset(input_ids_val, \n",
        "                             attention_masks_val, \n",
        "                             labels_val)"
      ],
      "metadata": {
        "id": "nwcRXfIT6ZnB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, we will create an iterator for our dataset using the torch DataLoader class. This will help save on memory during training and boost the training speed.\n",
        "batch_size = 32 for fine-tuned BERT, batch size of 16 and 32 are recommended.\n",
        "DataLoader iterates through our dataset.\n",
        "sampler = RandomSampler returns random indices."
      ],
      "metadata": {
        "id": "MQHn-nMd6gTi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler"
      ],
      "metadata": {
        "id": "WATrLjMT6dvi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 32\n",
        "dataloader_train = DataLoader(dataset_train, \n",
        "                              sampler = RandomSampler(dataset_train), \n",
        "                              batch_size = batch_size)\n",
        "\n",
        "dataloader_val = DataLoader(dataset_val, \n",
        "                              sampler = RandomSampler(dataset_val), \n",
        "                              batch_size = batch_size)"
      ],
      "metadata": {
        "id": "NUHJwgI66m4_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                 lr = 1e-5,\n",
        "                 eps = 1e-8) #2e-5 > 5e-5\n",
        "                 \n",
        "epochs = 10\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                           num_warmup_steps = 0,\n",
        "                                           num_training_steps = len(dataloader_train)*epochs)\n"
      ],
      "metadata": {
        "id": "nWOcQCJg6zLT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Argmax** is applied over the features and returns indices of predictions (a number between 0-5) which gives classes.\n",
        "**Flatten** flattens prediction by reshaping it into a one-dimensional tensor."
      ],
      "metadata": {
        "id": "Xd-vI4Ny-ee-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "def f1_score_func(preds, labels):\n",
        "    preds_flat = np.argmax(preds, axis = 1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return f1_score(labels_flat, preds, average = 'weighted')"
      ],
      "metadata": {
        "id": "Cd0DkQo26_nl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def accuracy_per_class(preds, labels):\n",
        "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    \n",
        "    for label in np.unique(labels_flat):\n",
        "        y_preds = preds_flat[labels_flat==label]\n",
        "        y_true = labels_flat[labels_flat==label]\n",
        "        print(f'Accuracy:{len(y_preds[y_preds==label])}/{len(y_true)}\\n')"
      ],
      "metadata": {
        "id": "3oAvGPFN-gCn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "seed_val = 17\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)"
      ],
      "metadata": {
        "id": "Pr2KViQVFObE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**To train model, put it in training mode by using .train() or .eval() for evaluations.**\n",
        "\n",
        "*   **tqdm** to track progress\n",
        "*   **to(device)** unpacks our data from the DataLoader and load the data onto \n",
        "the GPU\n",
        "*   **zero_grad** to set the gradient to 0.\n",
        "*   **input** will take in 3 values input_id, attention_mask, and labels\n",
        "*   We will train the model forward to get loss and logits\n",
        "*   **clip_grad_norm** clips the norm of the gradients to 1.0 to prevent exploding gradients\n",
        "*   **loss.backward()** performs backward pass to compute gradients\n",
        "*   **optimizer.step()** updates the model’s parameters\n",
        "*   **scheduler.step()** updates learning rate\n",
        "*   **evaluate and f1_score_func** to evaluate and compute loss and f1-score on the val set"
      ],
      "metadata": {
        "id": "FSQL9XphIOF_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(dataloader_val):\n",
        "\n",
        "    model.eval()\n",
        "    \n",
        "    loss_val_total = 0\n",
        "    predictions, true_vals = [], []\n",
        "    \n",
        "    for batch in tqdm(dataloader_val):\n",
        "        \n",
        "        inputs = {'input_ids':      batch[0],\n",
        "                  'attention_mask': batch[1],\n",
        "                  'labels':         batch[2]}\n",
        "\n",
        "        with torch.no_grad():        \n",
        "            outputs = model(**inputs)\n",
        "        \n",
        "        #compute loss\n",
        "        loss = outputs[0]\n",
        "        logits = outputs[1]\n",
        "        loss_val_total += loss.item()\n",
        "\n",
        "        #compute accuracy\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = inputs['labels'].cpu().numpy()\n",
        "        predictions.append(logits)\n",
        "        true_vals.append(label_ids)\n",
        "    \n",
        "    #compute average loss\n",
        "    loss_val_avg = loss_val_total/len(dataloader_val) \n",
        "    \n",
        "    predictions = np.concatenate(predictions, axis=0)\n",
        "    true_vals = np.concatenate(true_vals, axis=0)\n",
        "            \n",
        "    return loss_val_avg, predictions, true_vals"
      ],
      "metadata": {
        "id": "1Ry_NXfxKTBu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in tqdm(range(1, epochs+1)):\n",
        "    \n",
        "    model.train()\n",
        "    loss_train_total = 0\n",
        "    \n",
        "    progress_bar = tqdm(dataloader_train, \n",
        "                        desc = 'Epoch {:1d}'.format(epoch), \n",
        "                        leave = False, \n",
        "                        disable = False)\n",
        "    \n",
        "    for batch in progress_bar:\n",
        "         model.zero_grad()\n",
        "\n",
        "         inputs = {'input_ids': batch[0],\n",
        "                  'attention_mask': batch[1],\n",
        "                  'labels': batch[2]}\n",
        "         \n",
        "         outputs = model(**inputs)\n",
        "         loss= outputs[0]\n",
        "         loss_train_total += loss.item()\n",
        "         loss.backward()\n",
        "\n",
        "         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "         optimizer.step()\n",
        "         scheduler.step()\n",
        "         progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item() / len(batch))})\n",
        "\n",
        "    torch.save(model.state_dict(), path+f'/BERT_ft_epoch{epoch}.model')\n",
        "    \n",
        "    tqdm.write('\\n Epoch {epoch}')\n",
        "    \n",
        "    loss_train_ave = loss_train_total / len(dataloader_train)\n",
        "    tqdm.write('Training loss: {loss_train_avg}')\n",
        "    \n",
        "    val_loss, predictions, true_vals = evaluate(dataloader_val)\n",
        "    val_f1 = f1_score_func(predictions, true_vals)\n",
        "    tqdm.write(f'Validation loss: {val_loss}')\n",
        "    tqdm.write(f'F1 Score (weighted): {val_f1}')         "
      ],
      "metadata": {
        "id": "R2aPF_6dFrcM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# HuggingFace Custom Layers\n",
        "\n",
        "Say we have a small dataset about classifying some financial data in terms of sentiment. However, we go onto Huggingface and find that a lot of models have been trained for QA related to finance. We can use certain layers from these models for improving our own tasks.\n",
        "\n",
        "*Another simple example is when a certain domain-specific model has learned to classify text into 5 categories from a huge dataset it was trained on. Say we have a similar classification task, a completely different dataset in the same domain and only want to classify the data into 2 categories instead of 5. We can again use a model’s body and add our own head in an attempt to augment domain-specific knowledge on our own task.*\n"
      ],
      "metadata": {
        "id": "O1XQqObgTtJ0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset,Dataset,DatasetDict\n",
        "from transformers import DataCollatorWithPadding,AutoModelForSequenceClassification, Trainer, TrainingArguments,AutoTokenizer,AutoModel,AutoConfig\n",
        "from transformers.modeling_outputs import TokenClassifierOutput\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "XPj5xg_WToTj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 80% train, 20% test + validation\n",
        "train_testvalid = data.train_test_split(test_size=0.2,seed=15)\n",
        "\n",
        "# Split the 10% test + valid in half test, half valid\n",
        "test_valid = train_testvalid['test'].train_test_split(test_size=0.5,seed=15)\n",
        "\n",
        "# gather everyone if you want to have a single DatasetDict\n",
        "data = DatasetDict({\n",
        "    'train': train_testvalid['train'],\n",
        "    'test': test_valid['test'],\n",
        "    'valid': test_valid['train']})\n",
        "\n",
        "data"
      ],
      "metadata": {
        "id": "2gQUk8LCUgWF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = \"cardiffnlp/twitter-roberta-base-emotion\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "tokenizer.model_max_len=512"
      ],
      "metadata": {
        "id": "Y1Wi3xCvUvjX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize(batch):\n",
        "  return tokenizer(batch[\"headline\"], truncation=True,max_length=512)\n",
        "\n",
        "tokenized_dataset = data.map(tokenize, batched=True)\n",
        "tokenized_dataset"
      ],
      "metadata": {
        "id": "BVTTGzKCUxgH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_dataset.set_format(\"torch\",columns=[\"input_ids\", \"attention_mask\", \"label\"])\n",
        "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
      ],
      "metadata": {
        "id": "_uyTarA7U1za"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "First subclass the nn Module from PyTorch, extract the model body using AutoModel (from transformers) and provide the checkpoint to the model whose body we want to use. TokenClassifierOutput (from the transformers library) is returned which makes sure that our output is in a similar format to that from a Hugging Face model on the hub."
      ],
      "metadata": {
        "id": "Tgma-aaLVgCW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CustomModel(nn.Module):\n",
        "      def __init__(self,checkpoint,num_labels): \n",
        "        super(CustomModel,self).__init__() \n",
        "        self.num_labels = num_labels \n",
        "\n",
        "        #Load Model with given checkpoint and extract its body\n",
        "        self.model = model = AutoModel.from_pretrained(checkpoint,config=AutoConfig.from_pretrained(checkpoint, output_attentions=True,output_hidden_states=True))\n",
        "        self.dropout = nn.Dropout(0.1) \n",
        "        self.classifier = nn.Linear(768,num_labels) # load and initialize weights\n",
        "\n",
        "      def forward(self, input_ids=None, attention_mask=None,labels=None):\n",
        "        #Extract outputs from the body\n",
        "        outputs = self.model(input_ids=input_ids, attention_mask=attention_mask)\n",
        "\n",
        "        sequence_output = self.dropout(outputs[0]) #outputs[0] = last hidden state\n",
        "\n",
        "        logits = self.classifier(sequence_output[:,0,:].view(-1,768)) # calculate losses\n",
        "        \n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "          loss_fct = nn.CrossEntropyLoss()\n",
        "          loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n",
        "        \n",
        "        return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states,attentions=outputs.attentions)"
      ],
      "metadata": {
        "id": "0wBWk6RWU5zD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModel(checkpoint=checkpoint, num_labels=2)"
      ],
      "metadata": {
        "id": "dm1THD0tVAWS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "    tokenized_dataset[\"train\"], shuffle=True, batch_size=32, collate_fn=data_collator\n",
        ")\n",
        "eval_dataloader = DataLoader(\n",
        "    tokenized_dataset[\"valid\"], batch_size=32, collate_fn=data_collator\n",
        ")"
      ],
      "metadata": {
        "id": "6NjOoraDVVwr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AdamW,get_scheduler\n",
        "\n",
        "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
        "\n",
        "num_epochs = 3\n",
        "num_training_steps = num_epochs * len(train_dataloader)\n",
        "lr_scheduler = get_scheduler(\n",
        "    \"linear\",\n",
        "    optimizer=optimizer,\n",
        "    num_warmup_steps=0,\n",
        "    num_training_steps=num_training_steps,\n",
        ")\n",
        "print(num_training_steps)"
      ],
      "metadata": {
        "id": "pnboHGjOVrX3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_metric\n",
        "metric = load_metric(\"f1\")"
      ],
      "metadata": {
        "id": "cKaXpHBOV4EH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm.auto import tqdm\n",
        "\n",
        "progress_bar_train = tqdm(range(num_training_steps))\n",
        "progress_bar_eval = tqdm(range(num_epochs * len(eval_dataloader)))\n",
        "\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "  model.train()\n",
        "  for batch in train_dataloader:\n",
        "      batch = {k: v.to(device) for k, v in batch.items()}\n",
        "      outputs = model(**batch)\n",
        "      loss = outputs.loss\n",
        "      loss.backward()\n",
        "\n",
        "      optimizer.step()\n",
        "      lr_scheduler.step()\n",
        "      optimizer.zero_grad()\n",
        "      progress_bar_train.update(1)\n",
        "\n",
        "  model.eval()\n",
        "  for batch in eval_dataloader:\n",
        "    batch = {k: v.to(device) for k, v in batch.items()}\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**batch)\n",
        "\n",
        "    logits = outputs.logits\n",
        "    predictions = torch.argmax(logits, dim=-1)\n",
        "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
        "    progress_bar_eval.update(1)\n",
        "    \n",
        "  print(metric.compute())"
      ],
      "metadata": {
        "id": "4NOJs0snV5nt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.eval()\n",
        "\n",
        "test_dataloader = DataLoader(\n",
        "    tokenized_dataset[\"test\"], batch_size=32, collate_fn=data_collator\n",
        ")\n",
        "\n",
        "for batch in test_dataloader:\n",
        "    batch = {k: v.to(device) for k, v in batch.items()}\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**batch)\n",
        "\n",
        "    logits = outputs.logits\n",
        "    predictions = torch.argmax(logits, dim=-1)\n",
        "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
        "\n",
        "metric.compute()"
      ],
      "metadata": {
        "id": "WooeAkmiV-bZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This technique is particularly helpful in cases where we have small domain-specific datasets and want to leverage models trained on larger datasets in the same domain (task-agnostic) to augment performance on our small dataset."
      ],
      "metadata": {
        "id": "oXCGAKbDWDzk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Fine-Tuned BERT (Other)**"
      ],
      "metadata": {
        "id": "qxuYvqhcdIMp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In order to apply the pre-trained BERT, we must use the tokenizer provided by the library. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words."
      ],
      "metadata": {
        "id": "pbVA7g16dUle"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "metadata": {
        "id": "cxIGX4AfdN4D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    for sent in data:\n",
        "\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            #return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True      # Return attention mask\n",
        "            )\n",
        "        \n",
        "        # Add the outputs to the lists\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ],
      "metadata": {
        "id": "-zT8mFaKdWy1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "BERT-base consists of 12 transformer layers, each transformer layer takes in a list of token embeddings, and produces the same number of embeddings with the same hidden size (or dimensions) on the output. The output of the final transformer layer of the [CLS] token is used as the features of the sequence to feed a classifier."
      ],
      "metadata": {
        "id": "hzpb7IwidqPv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The transformers library has the BertForSequenceClassification class which is designed for classification tasks. However, we will create a new class so we can specify our own choice of classifiers.\n",
        "\n",
        "Below we will create a BertClassifier class with a BERT model to extract the last hidden layer of the [CLS] token and a single-hidden-layer feed-forward neural network as our classifier."
      ],
      "metadata": {
        "id": "Dlgjv-_Odq6y"
      }
    }
  ]
}