Example_HateExplain.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Example_HateExplain.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6d248353b8b04ebcaab705f0629a4438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6bffe5fef8d74bbaa31a61c0f1a32020",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_936b8a4513ac449ab0c77c2f912f1960",
              "IPY_MODEL_004b043ce4dc436ab1d8ee4f9fb6873b"
            ]
          }
        },
        "6bffe5fef8d74bbaa31a61c0f1a32020": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "936b8a4513ac449ab0c77c2f912f1960": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_89755305bd01495b8f54d6184d3cd844",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_01f24a559b8741aaa137adc5729a75a4"
          }
        },
        "004b043ce4dc436ab1d8ee4f9fb6873b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2c1575998e31458e811646dcaba8c759",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:04&lt;00:00,  4.51s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_091ff09479a74529a425affa22a500be"
          }
        },
        "89755305bd01495b8f54d6184d3cd844": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "01f24a559b8741aaa137adc5729a75a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c1575998e31458e811646dcaba8c759": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "091ff09479a74529a425affa22a500be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/punyajoy/HateXplain/blob/master/Example_HateExplain.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D5Hqb1OvsXDR",
        "outputId": "193dddd2-bbe5-46d9-bfdc-c3463d8a70b8"
      },
      "source": [
        "!git clone https://github.com/punyajoy/HateXplain.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'HateXplain'...\n",
            "remote: Enumerating objects: 73, done.\u001b[K\n",
            "remote: Counting objects: 100% (73/73), done.\u001b[K\n",
            "remote: Compressing objects: 100% (51/51), done.\u001b[K\n",
            "remote: Total 323 (delta 31), reused 49 (delta 21), pack-reused 250\u001b[K\n",
            "Receiving objects: 100% (323/323), 4.80 MiB | 21.08 MiB/s, done.\n",
            "Resolving deltas: 100% (172/172), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FPAJpqcYVEM0"
      },
      "source": [
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5l4M1_SHtDIm",
        "outputId": "0d99b5a0-4130-448c-ddd0-57b50e9c5481"
      },
      "source": [
        "cd HateXplain/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/HateXplain\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vl3PF9alWjHx"
      },
      "source": [
        "!mkdir Saved/\n",
        "!mkdir explanations_dicts/"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qU8P_dV5Wnk5",
        "outputId": "66230b91-3d65-4013-fcd1-040aa20b6706"
      },
      "source": [
        "!wget http://nlp.stanford.edu/data/glove.42B.300d.zip  -P Data/"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-22 09:58:06--  http://nlp.stanford.edu/data/glove.42B.300d.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.42B.300d.zip [following]\n",
            "--2020-12-22 09:58:06--  https://nlp.stanford.edu/data/glove.42B.300d.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: http://downloads.cs.stanford.edu/nlp/data/glove.42B.300d.zip [following]\n",
            "--2020-12-22 09:58:06--  http://downloads.cs.stanford.edu/nlp/data/glove.42B.300d.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1877800501 (1.7G) [application/zip]\n",
            "Saving to: ‘Data/glove.42B.300d.zip’\n",
            "\n",
            "glove.42B.300d.zip  100%[===================>]   1.75G  1.98MB/s    in 14m 37s \n",
            "\n",
            "2020-12-22 10:12:43 (2.04 MB/s) - ‘Data/glove.42B.300d.zip’ saved [1877800501/1877800501]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WZi9IlH-W_83",
        "outputId": "b46a5a0b-035d-4506-a1b8-8dca2590236f"
      },
      "source": [
        "!unzip Data/glove.42B.300d.zip -d Data/"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  Data/glove.42B.300d.zip\n",
            "  inflating: Data/glove.42B.300d.txt  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gofRAO-crNgI"
      },
      "source": [
        "!rm Data/glove.42B.300d.zip"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "collapsed": true,
        "id": "dkIaNCQDgvSP",
        "outputId": "c7e18222-4408-4ca1-9af7-b944f5c90fd1"
      },
      "source": [
        "!pip install -r requirements.txt"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (1.4.1)\n",
            "Collecting spacy==2.3.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/b5/c7a92c7ce5d4b353b70b4b5b4385687206c8b230ddfe08746ab0fd310a3a/spacy-2.3.2-cp36-cp36m-manylinux1_x86_64.whl (9.9MB)\n",
            "\u001b[K     |████████████████████████████████| 10.0MB 11.7MB/s \n",
            "\u001b[?25hCollecting tqdm==4.43.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/47/55/fd9170ba08a1a64a18a7f8a18f088037316f2a41be04d2fe6ece5a653e8f/tqdm-4.43.0-py2.py3-none-any.whl (59kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 10.4MB/s \n",
            "\u001b[?25hCollecting Keras==2.3.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ad/fd/6bfe87920d7f4fd475acd28500a42482b6b84479832bdc0fe9e589a60ceb/Keras-2.3.1-py2.py3-none-any.whl (377kB)\n",
            "\u001b[K     |████████████████████████████████| 378kB 62.9MB/s \n",
            "\u001b[?25hCollecting waiting==1.4.1\n",
            "  Downloading https://files.pythonhosted.org/packages/0b/db/51392c77d22acef400d8e9e245aaed94919e937f7edbc508865351f3e973/waiting-1.4.1.tar.gz\n",
            "Collecting ekphrasis==0.5.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/92/e6/37c59d65e78c3a2aaf662df58faca7250eb6b36c559b912a39a7ca204cfb/ekphrasis-0.5.1.tar.gz (80kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 14.1MB/s \n",
            "\u001b[?25hCollecting pandas==1.0.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/71/8f53bdbcbc67c912b888b40def255767e475402e9df64050019149b1a943/pandas-1.0.3-cp36-cp36m-manylinux1_x86_64.whl (10.0MB)\n",
            "\u001b[K     |████████████████████████████████| 10.0MB 43.9MB/s \n",
            "\u001b[?25hCollecting transformers==2.5.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/33/ffb67897a6985a7b7d8e5e7878c3628678f553634bd3836404fef06ef19b/transformers-2.5.1-py3-none-any.whl (499kB)\n",
            "\u001b[K     |████████████████████████████████| 501kB 54.8MB/s \n",
            "\u001b[?25hCollecting lime==0.2.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/86/91a13127d83d793ecb50eb75e716f76e6eda809b6803c5a4ff462339789e/lime-0.2.0.1.tar.gz (275kB)\n",
            "\u001b[K     |████████████████████████████████| 276kB 61.1MB/s \n",
            "\u001b[?25hCollecting numpy==1.16.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c1/e2/4db8df8f6cddc98e7d7c537245ef2f4e41a1ed17bf0c3177ab3cc6beac7f/numpy-1.16.3-cp36-cp36m-manylinux1_x86_64.whl (17.3MB)\n",
            "\u001b[K     |████████████████████████████████| 17.3MB 164kB/s \n",
            "\u001b[?25hCollecting matplotlib==3.2.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/93/4b/52da6b1523d5139d04e02d9e26ceda6146b48f2a4e5d2abfdf1c7bac8c40/matplotlib-3.2.1-cp36-cp36m-manylinux1_x86_64.whl (12.4MB)\n",
            "\u001b[K     |████████████████████████████████| 12.4MB 55.9MB/s \n",
            "\u001b[?25hCollecting gensim==3.8.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/dd/112bd4258cee11e0baaaba064060eb156475a42362e59e3ff28e7ca2d29d/gensim-3.8.1-cp36-cp36m-manylinux1_x86_64.whl (24.2MB)\n",
            "\u001b[K     |████████████████████████████████| 24.2MB 143kB/s \n",
            "\u001b[?25hCollecting neptune_client==0.4.107\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f4/a4/a3a3bda2d8ad7fa7d3e824ef0f6779167f657d85224fec5c293f652de98e/neptune-client-0.4.107.tar.gz (86kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 15.2MB/s \n",
            "\u001b[?25hCollecting knockknock==0.1.7\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/d0/435a247d19cbce8cd88560776c45e1472b02bbf008821a8a11d5afae2ef8/knockknock-0.1.7-py3-none-any.whl\n",
            "Collecting torch==1.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/60/f685fb2cfb3088736bafbc9bdbb455327bdc8906b606da9c9a81bae1c81e/torch-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (676.9MB)\n",
            "\u001b[K     |████████████████████████████████| 676.9MB 15kB/s \n",
            "\u001b[?25hCollecting apex==0.9.10dev\n",
            "  Downloading https://files.pythonhosted.org/packages/31/b6/923de12ffcc2686157d7f74b96396b87c854eaaeec9d441d120facc2a0e0/apex-0.9.10dev.tar.gz\n",
            "Requirement already satisfied: dataclasses==0.8 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 17)) (0.8)\n",
            "Collecting GPUtil==1.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz\n",
            "Collecting scikit_learn==0.23.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5c/a1/273def87037a7fb010512bbc5901c31cfddfca8080bc63b42b26e3cc55b3/scikit_learn-0.23.2-cp36-cp36m-manylinux1_x86_64.whl (6.8MB)\n",
            "\u001b[K     |████████████████████████████████| 6.8MB 39.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (0.8.0)\n",
            "Collecting thinc==7.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/ae/ef3ae5e93639c0ef8e3eb32e3c18341e511b3c515fcfc603f4b808087651/thinc-7.4.1-cp36-cp36m-manylinux1_x86_64.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 51.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (1.0.5)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (2.0.5)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (1.0.0)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (50.3.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (2.23.0)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (1.1.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (3.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.3.2->-r requirements.txt (line 2)) (1.0.5)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1->-r requirements.txt (line 4)) (2.10.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1->-r requirements.txt (line 4)) (1.1.2)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1->-r requirements.txt (line 4)) (1.15.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1->-r requirements.txt (line 4)) (3.13)\n",
            "Collecting keras-applications>=1.0.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from ekphrasis==0.5.1->-r requirements.txt (line 6)) (1.1.0)\n",
            "Collecting colorama\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Collecting ujson\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f1/84/e039c6ffc6603f2dfe966972d345d4f650a4ffd74b18c852ece645de12ac/ujson-4.0.1-cp36-cp36m-manylinux1_x86_64.whl (179kB)\n",
            "\u001b[K     |████████████████████████████████| 184kB 66.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from ekphrasis==0.5.1->-r requirements.txt (line 6)) (3.2.5)\n",
            "Collecting ftfy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/e2/3b51c53dffb1e52d9210ebc01f1fb9f2f6eba9b3201fa971fd3946643c71/ftfy-5.8.tar.gz (64kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 12.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas==1.0.3->-r requirements.txt (line 7)) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas==1.0.3->-r requirements.txt (line 7)) (2.8.1)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 55.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.5.1->-r requirements.txt (line 8)) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 56.7MB/s \n",
            "\u001b[?25hCollecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c9/c1/5c2b2259dd4a149f873f1ab9b4c5ef106c828a4abc7230c9452be8c27493/boto3-1.16.41-py2.py3-none-any.whl (130kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 63.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.5.1->-r requirements.txt (line 8)) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 55.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-image>=0.12 in /usr/local/lib/python3.6/dist-packages (from lime==0.2.0.1->-r requirements.txt (line 9)) (0.16.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.1->-r requirements.txt (line 11)) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.1->-r requirements.txt (line 11)) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.1->-r requirements.txt (line 11)) (1.3.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.6/dist-packages (from gensim==3.8.1->-r requirements.txt (line 12)) (4.0.1)\n",
            "Collecting bravado\n",
            "  Downloading https://files.pythonhosted.org/packages/6d/3d/f8772d9295c03e08a9ab4afc1ccd195efe6cb4d1af3135b7f74eb8beb0d6/bravado-11.0.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from neptune_client==0.4.107->-r requirements.txt (line 13)) (7.1.2)\n",
            "Collecting future>=0.17.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n",
            "\u001b[K     |████████████████████████████████| 829kB 52.3MB/s \n",
            "\u001b[?25hCollecting py3nvml\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/53/b3/cb30dd8cc1198ae3fdb5a320ca7986d7ca76e23d16415067eafebff8685f/py3nvml-0.2.6-py3-none-any.whl (55kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 10.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: oauthlib>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from neptune_client==0.4.107->-r requirements.txt (line 13)) (3.1.0)\n",
            "Requirement already satisfied: Pillow>=1.1.6 in /usr/local/lib/python3.6/dist-packages (from neptune_client==0.4.107->-r requirements.txt (line 13)) (7.0.0)\n",
            "Collecting PyJWT\n",
            "  Downloading https://files.pythonhosted.org/packages/87/8b/6a9f14b5f781697e51259d81657e6048fd31a113229cf346880bb7545565/PyJWT-1.7.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: requests-oauthlib>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from neptune_client==0.4.107->-r requirements.txt (line 13)) (1.3.0)\n",
            "Collecting websocket-client>=0.35.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/5f/f61b420143ed1c8dc69f9eaec5ff1ac36109d52c80de49d66e0c36c3dfdf/websocket_client-0.57.0-py2.py3-none-any.whl (200kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 65.3MB/s \n",
            "\u001b[?25hCollecting GitPython>=2.0.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/d1/a7f8fe3df258549b303415157328bfcc63e9b11d06a7ad7a3327f3d32606/GitPython-3.1.11-py3-none-any.whl (159kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 64.4MB/s \n",
            "\u001b[?25hCollecting python-telegram-bot\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cc/b3/f363e9c5c2e4690a1fd352c01263eb2938952888c09d73c824b49d288dcc/python_telegram_bot-13.1-py3-none-any.whl (422kB)\n",
            "\u001b[K     |████████████████████████████████| 430kB 53.4MB/s \n",
            "\u001b[?25hCollecting keyring\n",
            "  Downloading https://files.pythonhosted.org/packages/53/14/1c952bcd21255f42f9ba0280d3abd8074dca2c27d136eb749b98ab478f72/keyring-21.5.0-py3-none-any.whl\n",
            "Collecting yagmail>=0.11.214\n",
            "  Downloading https://files.pythonhosted.org/packages/94/79/4cdc548dd49821a3fb87bb39b05d262e347f37dc331bc2c45f1a90856712/yagmail-0.14.245-py2.py3-none-any.whl\n",
            "Collecting matrix-client\n",
            "  Downloading https://files.pythonhosted.org/packages/6b/0b/65dc841fd3d14e7ebc6081bbfce23365a6b2f68cc6ae2ae2d1d7d59570cd/matrix_client-0.3.2-py2.py3-none-any.whl\n",
            "Collecting twilio\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e9/0e/d54630e6daae43dd74d44a94f52d1072b5332c374d699938d7d1db20a54c/twilio-6.50.1.tar.gz (457kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 57.1MB/s \n",
            "\u001b[?25hCollecting cryptacular\n",
            "  Downloading https://files.pythonhosted.org/packages/ec/d6/a82d191ec058314b2b7cbee5635150f754ba1c6ffc05387bc9a57efe48b8/cryptacular-1.5.5.tar.gz\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting zope.sqlalchemy\n",
            "  Downloading https://files.pythonhosted.org/packages/fa/83/459decec1dd2c14d60f9a360fff989c128abe545a1554a1da64b054a55d4/zope.sqlalchemy-1.3-py2.py3-none-any.whl\n",
            "Collecting velruse>=1.0.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/d9/e18b5c98667c45f5dd1a256d72168ea5ff68f0025fc5b24be010f2696ca3/velruse-1.1.1.tar.gz (709kB)\n",
            "\u001b[K     |████████████████████████████████| 716kB 57.1MB/s \n",
            "\u001b[?25hCollecting pyramid>1.1.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1b/5a/6f934f9cf154aaf74469c2665029b473bb553ac0e1c9aa25f6d4d7891333/pyramid-1.10.5-py2.py3-none-any.whl (326kB)\n",
            "\u001b[K     |████████████████████████████████| 327kB 60.4MB/s \n",
            "\u001b[?25hCollecting pyramid_mailer\n",
            "  Downloading https://files.pythonhosted.org/packages/ea/c3/0ce593179a8da8e1ab7fe178b0ae096a046246bd44a5787f72940d6dd5b2/pyramid_mailer-0.15.1-py2.py3-none-any.whl\n",
            "Collecting wtforms\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e0/31/614fc7dc7d76005b0acb8c0c8920d962b83d7422b4ba912886dfb63f86ff/WTForms-2.3.3-py2.py3-none-any.whl (169kB)\n",
            "\u001b[K     |████████████████████████████████| 174kB 59.3MB/s \n",
            "\u001b[?25hCollecting wtforms-recaptcha\n",
            "  Downloading https://files.pythonhosted.org/packages/5c/b0/42021ab061b768e3e5f430466219468c2afec99fe706e4340792d7a6fab4/wtforms_recaptcha-0.3.2-py2.py3-none-any.whl\n",
            "Collecting threadpoolctl>=2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit_learn==0.23.2->-r requirements.txt (line 19)) (1.0.0)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy==2.3.2->-r requirements.txt (line 2)) (3.3.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.3.2->-r requirements.txt (line 2)) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.3.2->-r requirements.txt (line 2)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.3.2->-r requirements.txt (line 2)) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.3.2->-r requirements.txt (line 2)) (2.10)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from ftfy->ekphrasis==0.5.1->-r requirements.txt (line 6)) (0.2.5)\n",
            "Collecting s3transfer<0.4.0,>=0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 12.7MB/s \n",
            "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Collecting botocore<1.20.0,>=1.19.41\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b9/69/eecc498592e2ee9a300037881b38637358dbddd5211d2af061c8b177abe4/botocore-1.19.41-py2.py3-none-any.whl (7.1MB)\n",
            "\u001b[K     |████████████████████████████████| 7.2MB 51.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: imageio>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image>=0.12->lime==0.2.0.1->-r requirements.txt (line 9)) (2.4.1)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image>=0.12->lime==0.2.0.1->-r requirements.txt (line 9)) (1.1.1)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image>=0.12->lime==0.2.0.1->-r requirements.txt (line 9)) (2.5)\n",
            "Collecting monotonic\n",
            "  Downloading https://files.pythonhosted.org/packages/ac/aa/063eca6a416f397bd99552c534c6d11d57f58f2e94c14780f3bbf818c4cf/monotonic-1.5-py2.py3-none-any.whl\n",
            "Collecting simplejson\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/73/96/1e6b19045375890068d7342cbe280dd64ae73fd90b9735b5efb8d1e044a1/simplejson-3.17.2-cp36-cp36m-manylinux2010_x86_64.whl (127kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 63.6MB/s \n",
            "\u001b[?25hCollecting bravado-core>=5.16.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/11/18e9d28a156c33f2d5f15a5e155dc7130250acb0a569255a2b6b307b596d/bravado_core-5.17.0-py2.py3-none-any.whl (67kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 12.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from bravado->neptune_client==0.4.107->-r requirements.txt (line 13)) (3.7.4.3)\n",
            "Requirement already satisfied: msgpack in /usr/local/lib/python3.6/dist-packages (from bravado->neptune_client==0.4.107->-r requirements.txt (line 13)) (1.0.1)\n",
            "Collecting xmltodict\n",
            "  Downloading https://files.pythonhosted.org/packages/28/fd/30d5c1d3ac29ce229f6bdc40bbc20b28f716e8b363140c26eff19122d8a5/xmltodict-0.12.0-py2.py3-none-any.whl\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 12.9MB/s \n",
            "\u001b[?25hCollecting cryptography\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c9/de/7054df0620b5411ba45480f0261e1fb66a53f3db31b28e3aa52c026e72d9/cryptography-3.3.1-cp36-abi3-manylinux2010_x86_64.whl (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6MB 44.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tornado>=5.1 in /usr/local/lib/python3.6/dist-packages (from python-telegram-bot->knockknock==0.1.7->-r requirements.txt (line 14)) (5.1.1)\n",
            "Collecting APScheduler==3.6.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f3/34/9ef20ed473c4fd2c3df54ef77a27ae3fc7500b16b192add4720cab8b2c09/APScheduler-3.6.3-py2.py3-none-any.whl (58kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 6.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: decorator>=4.4.0 in /usr/local/lib/python3.6/dist-packages (from python-telegram-bot->knockknock==0.1.7->-r requirements.txt (line 14)) (4.4.2)\n",
            "Collecting jeepney>=0.4.2; sys_platform == \"linux\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/b0/a6ea72741aaac3f37fb96d195e4ee576a103c4c04e279bc6b446a70960e1/jeepney-0.6.0-py3-none-any.whl (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 9.4MB/s \n",
            "\u001b[?25hCollecting SecretStorage>=3.2; sys_platform == \"linux\"\n",
            "  Downloading https://files.pythonhosted.org/packages/63/a2/a6d9099b14eb5dbbb04fb722d2b5322688f8f99b471bdf2097e33efa8091/SecretStorage-3.3.0-py3-none-any.whl\n",
            "Collecting premailer\n",
            "  Downloading https://files.pythonhosted.org/packages/cd/ce/74bbdf0eee4265fd3f161d4276b36c9238b802191c2053c8e68578bda4e6/premailer-3.7.0-py2.py3-none-any.whl\n",
            "Collecting pbkdf2\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c0/6a2376ae81beb82eda645a091684c0b0becb86b972def7849ea9066e3d5e/pbkdf2-1.3.tar.gz\n",
            "Collecting transaction>=1.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9b/35/b5cca132f9b364066bea00cbf4ea466b7a15461a609ab9ba8e832e165452/transaction-3.0.1-py2.py3-none-any.whl (47kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: SQLAlchemy>=0.7 in /usr/local/lib/python3.6/dist-packages (from zope.sqlalchemy->apex==0.9.10dev->-r requirements.txt (line 16)) (1.3.20)\n",
            "Collecting zope.interface>=3.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/82/b0/da8afd9b3bd50c7665ecdac062f182982af1173c9081f9af7261091c5588/zope.interface-5.2.0-cp36-cp36m-manylinux2010_x86_64.whl (236kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 61.7MB/s \n",
            "\u001b[?25hCollecting anykeystore\n",
            "  Downloading https://files.pythonhosted.org/packages/aa/dc/c4399c0e6b835710763705220f9c37681683f950678db799a5c7eda9e154/anykeystore-0.2.tar.gz\n",
            "Collecting python3-openid\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e0/a5/c6ba13860bdf5525f1ab01e01cc667578d6f1efc8a1dba355700fb04c29b/python3_openid-3.2.0-py3-none-any.whl (133kB)\n",
            "\u001b[K     |████████████████████████████████| 143kB 59.2MB/s \n",
            "\u001b[?25hCollecting plaster\n",
            "  Downloading https://files.pythonhosted.org/packages/61/29/3ac8a5d03b2d9e6b876385066676472ba4acf93677acfc7360b035503d49/plaster-1.0-py2.py3-none-any.whl\n",
            "Collecting venusian>=1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/43/92/3d522a710867168ee422a0ffbd712c425ece937aaeec4381497a59e24faf/venusian-3.0.0-py3-none-any.whl\n",
            "Collecting translationstring>=0.4\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/98/36187601a15e3d37e9bfcf0e0e1055532b39d044353b06861c3a519737a9/translationstring-1.4-py2.py3-none-any.whl\n",
            "Collecting plaster-pastedeploy\n",
            "  Downloading https://files.pythonhosted.org/packages/11/c4/0470056ea324c7a420c22647be512dec1b5e32b1b6e77e27c61838d2811c/plaster_pastedeploy-0.7-py2.py3-none-any.whl\n",
            "Collecting webob>=1.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/3c/de37900faff3c95c7d55dd557aa71bd77477950048983dcd4b53f96fde40/WebOb-1.8.6-py2.py3-none-any.whl (114kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 63.6MB/s \n",
            "\u001b[?25hCollecting hupper>=1.5\n",
            "  Downloading https://files.pythonhosted.org/packages/48/7f/06ace28143b2cb3a4b14c9d9e5165741d2d133ef331b616acf47ab5c3517/hupper-1.10.2-py2.py3-none-any.whl\n",
            "Collecting zope.deprecation>=3.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f9/26/b935bbf9d27e898b87d80e7873a0200cebf239253d0afe7a59f82fe90fff/zope.deprecation-4.4.0-py2.py3-none-any.whl\n",
            "Collecting repoze.sendmail>=4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/25/98/c5c64dc045b7c45858c391d04673a0f2748acef8e0eea4f2989b22220f97/repoze.sendmail-4.4.1-py2.py3-none-any.whl (41kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 10.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe in /usr/local/lib/python3.6/dist-packages (from wtforms->apex==0.9.10dev->-r requirements.txt (line 16)) (1.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy==2.3.2->-r requirements.txt (line 2)) (3.4.0)\n",
            "Requirement already satisfied: jsonschema[format]>=2.5.1 in /usr/local/lib/python3.6/dist-packages (from bravado-core>=5.16.1->bravado->neptune_client==0.4.107->-r requirements.txt (line 13)) (2.6.0)\n",
            "Collecting jsonref\n",
            "  Downloading https://files.pythonhosted.org/packages/07/92/f8e4ac824b14af77e613984e480fa818397c72d4141fc466decb26752749/jsonref-0.2-py3-none-any.whl\n",
            "Collecting swagger-spec-validator>=2.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/09/de/e78cefbf5838b434b63a789264b79821cb2267f1498fbed23ef8590133e4/swagger_spec_validator-2.7.3-py2.py3-none-any.whl\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.6/dist-packages (from cryptography->python-telegram-bot->knockknock==0.1.7->-r requirements.txt (line 14)) (1.14.4)\n",
            "Requirement already satisfied: tzlocal>=1.2 in /usr/local/lib/python3.6/dist-packages (from APScheduler==3.6.3->python-telegram-bot->knockknock==0.1.7->-r requirements.txt (line 14)) (1.5.1)\n",
            "Collecting cssutils\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6b/15/a9fb9010f58d1c55dd0b7779db2334feb9a572d407024f39a60f44293861/cssutils-1.0.2-py3-none-any.whl (406kB)\n",
            "\u001b[K     |████████████████████████████████| 409kB 65.0MB/s \n",
            "\u001b[?25hCollecting cssselect\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/d4/3b5c17f00cce85b9a1e6f91096e1cc8e8ede2e1be8e96b87ce1ed09e92c5/cssselect-1.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: cachetools in /usr/local/lib/python3.6/dist-packages (from premailer->yagmail>=0.11.214->knockknock==0.1.7->-r requirements.txt (line 14)) (4.2.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.6/dist-packages (from premailer->yagmail>=0.11.214->knockknock==0.1.7->-r requirements.txt (line 14)) (4.2.6)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from python3-openid->velruse>=1.0.3->apex==0.9.10dev->-r requirements.txt (line 16)) (0.6.0)\n",
            "Collecting PasteDeploy>=2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/8f/0b/d47ea894587f3155f8c4520aa74d57c856189d0bbe27e831881d655a3386/PasteDeploy-2.1.1-py2.py3-none-any.whl\n",
            "Collecting rfc3987; extra == \"format\"\n",
            "  Downloading https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl\n",
            "Collecting webcolors; extra == \"format\"\n",
            "  Downloading https://files.pythonhosted.org/packages/12/05/3350559de9714b202e443a9e6312937341bd5f79f4e4f625744295e7dd17/webcolors-1.11.1-py3-none-any.whl\n",
            "Collecting strict-rfc3339; extra == \"format\"\n",
            "  Downloading https://files.pythonhosted.org/packages/56/e4/879ef1dbd6ddea1c77c0078cd59b503368b0456bcca7d063a870ca2119d3/strict-rfc3339-0.7.tar.gz\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.12->cryptography->python-telegram-bot->knockknock==0.1.7->-r requirements.txt (line 14)) (2.20)\n",
            "Building wheels for collected packages: cryptacular\n",
            "  Building wheel for cryptacular (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cryptacular: filename=cryptacular-1.5.5-cp36-abi3-manylinux2010_x86_64.whl size=48031 sha256=bcb9c49149a66c882aa757244a47f5ae16546983c1c985a6318620a3234a8f86\n",
            "  Stored in directory: /root/.cache/pip/wheels/3c/79/bc/1eec7120c3ff9b0a2c7ad94d1626abc3388688e2ed7a45878f\n",
            "Successfully built cryptacular\n",
            "Building wheels for collected packages: waiting, ekphrasis, lime, neptune-client, apex, GPUtil, ftfy, sacremoses, future, twilio, velruse, pbkdf2, anykeystore, strict-rfc3339\n",
            "  Building wheel for waiting (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for waiting: filename=waiting-1.4.1-cp36-none-any.whl size=3763 sha256=d8033beff0666ce6d10d6e009695937e720bcd7ab2d9b14ad49a33c63d0eb91f\n",
            "  Stored in directory: /root/.cache/pip/wheels/79/cf/8e/9cb9b856303e68772eb193cc9906b8cff95a595b5df0586c33\n",
            "  Building wheel for ekphrasis (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ekphrasis: filename=ekphrasis-0.5.1-cp36-none-any.whl size=82844 sha256=3b06c1e0963387bb6417759698a9d4f27c6433be91b8c3fbc30e23c5bf12ea12\n",
            "  Stored in directory: /root/.cache/pip/wheels/2f/c5/9b/c9b60f535a2cf9fdbc92d84c4801a010c35a9cd348011ed2a1\n",
            "  Building wheel for lime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lime: filename=lime-0.2.0.1-cp36-none-any.whl size=283846 sha256=3a87125f5be9736d34164aa157a4c45f9bc360608e511e162dcfa29f350fb0ff\n",
            "  Stored in directory: /root/.cache/pip/wheels/4c/4f/a5/0bc765457bd41378bf3ce8d17d7495369d6e7ca3b712c60c89\n",
            "  Building wheel for neptune-client (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for neptune-client: filename=neptune_client-0.4.107-py2.py3-none-any.whl size=145061 sha256=c21781e400b84c4f3788630cd37f51cb038537f6ef7eb7d7a6248ede35ea0ed1\n",
            "  Stored in directory: /root/.cache/pip/wheels/76/05/e9/6329662e775aa3537466339ebde8ce2ee76cb3296dea1c04ec\n",
            "  Building wheel for apex (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for apex: filename=apex-0.9.10.dev0-cp36-none-any.whl size=46468 sha256=997b72da737c1df2cd79e102fba270840a0cd7b8524fd4c68a1735dbbec094a2\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/4c/4b/2990cf86a29c679ae4bd5f4de5723aa8a4af107721089c9a55\n",
            "  Building wheel for GPUtil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPUtil: filename=GPUtil-1.4.0-cp36-none-any.whl size=7411 sha256=6b07357bc2b0fd4e67ec9c76c0934ce5600d6761a8e34c6b665a233c74b7d2b1\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ftfy: filename=ftfy-5.8-cp36-none-any.whl size=45613 sha256=dc4515f36b30858ca9cb01b2f0db562e382efd95b878e200cd8668ce63a4df26\n",
            "  Stored in directory: /root/.cache/pip/wheels/ba/c0/ef/f28c4da5ac84a4e06ac256ca9182fc34fa57fefffdbc68425b\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=df52f8f8e66e90b49caf0931730fa18274b7d59d3277943b93d9e9be7763fab1\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for future: filename=future-0.18.2-cp36-none-any.whl size=491057 sha256=a0149eb6525b623f5c804ca02251916f9fa41a46ec9acaa90f381bc04d2938a1\n",
            "  Stored in directory: /root/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n",
            "  Building wheel for twilio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for twilio: filename=twilio-6.50.1-py2.py3-none-any.whl size=1208685 sha256=e253d05ffeff1a20a74ebfc518ef31c0b0c067e4164aa01942b35503bf6ba481\n",
            "  Stored in directory: /root/.cache/pip/wheels/17/10/6c/1b04371d399b059dcea195e00729e096fd959e1e35b0e7c8a2\n",
            "  Building wheel for velruse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for velruse: filename=velruse-1.1.1-cp36-none-any.whl size=50923 sha256=4af5d403bbcdc06cec221a33ebe5bece10ecf0b92c08a4f6a8514cce95665e69\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/65/9e/b805aad8ec3a359591c497b257dabe911f305d285b5d8a13cc\n",
            "  Building wheel for pbkdf2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pbkdf2: filename=pbkdf2-1.3-cp36-none-any.whl size=5103 sha256=5f49a90b52e2d974780fb690354f8ca38df73f6bb8dc68e84c27afd31bb43c02\n",
            "  Stored in directory: /root/.cache/pip/wheels/8e/62/b9/0bf3a68f2111e169253ec4d2bbdc303c46777b7fc99bbbf230\n",
            "  Building wheel for anykeystore (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for anykeystore: filename=anykeystore-0.2-cp36-none-any.whl size=17026 sha256=5d916dfbcf28dbb8d4d9b341f2ffd5500606d8d104d5d11812e9c843a18c7ff3\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/a9/b2/f79f84cbee6613c9edce6d98b9e1410c1d41d38953bd94eed2\n",
            "  Building wheel for strict-rfc3339 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for strict-rfc3339: filename=strict_rfc3339-0.7-cp36-none-any.whl size=18122 sha256=1a15a257c1954c93b5b0973fbe595aab9691cbd20ceebbc608775ad63e507286\n",
            "  Stored in directory: /root/.cache/pip/wheels/bb/af/c9/b6e9fb5f9b2470e4ed2a7241c9ab3a8cdd3bc8555ae02ca2e6\n",
            "Successfully built waiting ekphrasis lime neptune-client apex GPUtil ftfy sacremoses future twilio velruse pbkdf2 anykeystore strict-rfc3339\n",
            "\u001b[31mERROR: umap-learn 0.4.6 has requirement numpy>=1.17, but you'll have numpy 1.16.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: torchvision 0.8.1+cu101 has requirement torch==1.7.0, but you'll have torch 1.1.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow 2.4.0 has requirement numpy~=1.19.2, but you'll have numpy 1.16.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement pandas~=1.1.0; python_version >= \"3.0\", but you'll have pandas 1.0.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: fbprophet 0.7.1 has requirement pandas>=1.0.4, but you'll have pandas 1.0.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: botocore 1.19.41 has requirement urllib3<1.27,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tqdm, numpy, thinc, spacy, keras-applications, Keras, waiting, colorama, ujson, matplotlib, ftfy, ekphrasis, pandas, tokenizers, sentencepiece, jmespath, botocore, s3transfer, boto3, sacremoses, transformers, threadpoolctl, scikit-learn, lime, gensim, monotonic, simplejson, jsonref, swagger-spec-validator, bravado-core, bravado, future, xmltodict, py3nvml, PyJWT, websocket-client, smmap, gitdb, GitPython, neptune-client, cryptography, APScheduler, python-telegram-bot, jeepney, SecretStorage, keyring, cssutils, cssselect, premailer, yagmail, matrix-client, twilio, knockknock, torch, pbkdf2, cryptacular, zope.interface, transaction, zope.sqlalchemy, plaster, venusian, translationstring, PasteDeploy, plaster-pastedeploy, webob, hupper, zope.deprecation, pyramid, anykeystore, python3-openid, velruse, repoze.sendmail, pyramid-mailer, wtforms, wtforms-recaptcha, apex, GPUtil, rfc3987, webcolors, strict-rfc3339\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "  Found existing installation: numpy 1.19.4\n",
            "    Uninstalling numpy-1.19.4:\n",
            "      Successfully uninstalled numpy-1.19.4\n",
            "  Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Found existing installation: spacy 2.2.4\n",
            "    Uninstalling spacy-2.2.4:\n",
            "      Successfully uninstalled spacy-2.2.4\n",
            "  Found existing installation: Keras 2.4.3\n",
            "    Uninstalling Keras-2.4.3:\n",
            "      Successfully uninstalled Keras-2.4.3\n",
            "  Found existing installation: matplotlib 3.2.2\n",
            "    Uninstalling matplotlib-3.2.2:\n",
            "      Successfully uninstalled matplotlib-3.2.2\n",
            "  Found existing installation: pandas 1.1.5\n",
            "    Uninstalling pandas-1.1.5:\n",
            "      Successfully uninstalled pandas-1.1.5\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "  Found existing installation: gensim 3.6.0\n",
            "    Uninstalling gensim-3.6.0:\n",
            "      Successfully uninstalled gensim-3.6.0\n",
            "  Found existing installation: future 0.16.0\n",
            "    Uninstalling future-0.16.0:\n",
            "      Successfully uninstalled future-0.16.0\n",
            "  Found existing installation: torch 1.7.0+cu101\n",
            "    Uninstalling torch-1.7.0+cu101:\n",
            "      Successfully uninstalled torch-1.7.0+cu101\n",
            "Successfully installed APScheduler-3.6.3 GPUtil-1.4.0 GitPython-3.1.11 Keras-2.3.1 PasteDeploy-2.1.1 PyJWT-1.7.1 SecretStorage-3.3.0 anykeystore-0.2 apex-0.9.10.dev0 boto3-1.16.41 botocore-1.19.41 bravado-11.0.2 bravado-core-5.17.0 colorama-0.4.4 cryptacular-1.5.5 cryptography-3.3.1 cssselect-1.1.0 cssutils-1.0.2 ekphrasis-0.5.1 ftfy-5.8 future-0.18.2 gensim-3.8.1 gitdb-4.0.5 hupper-1.10.2 jeepney-0.6.0 jmespath-0.10.0 jsonref-0.2 keras-applications-1.0.8 keyring-21.5.0 knockknock-0.1.7 lime-0.2.0.1 matplotlib-3.2.1 matrix-client-0.3.2 monotonic-1.5 neptune-client-0.4.107 numpy-1.16.3 pandas-1.0.3 pbkdf2-1.3 plaster-1.0 plaster-pastedeploy-0.7 premailer-3.7.0 py3nvml-0.2.6 pyramid-1.10.5 pyramid-mailer-0.15.1 python-telegram-bot-13.1 python3-openid-3.2.0 repoze.sendmail-4.4.1 rfc3987-1.3.8 s3transfer-0.3.3 sacremoses-0.0.43 scikit-learn-0.23.2 sentencepiece-0.1.94 simplejson-3.17.2 smmap-3.0.4 spacy-2.3.2 strict-rfc3339-0.7 swagger-spec-validator-2.7.3 thinc-7.4.1 threadpoolctl-2.1.0 tokenizers-0.5.2 torch-1.1.0 tqdm-4.43.0 transaction-3.0.1 transformers-2.5.1 translationstring-1.4 twilio-6.50.1 ujson-4.0.1 velruse-1.1.1 venusian-3.0.0 waiting-1.4.1 webcolors-1.11.1 webob-1.8.6 websocket-client-0.57.0 wtforms-2.3.3 wtforms-recaptcha-0.3.2 xmltodict-0.12.0 yagmail-0.14.245 zope.deprecation-4.4.0 zope.interface-5.2.0 zope.sqlalchemy-1.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "matplotlib",
                  "mpl_toolkits",
                  "numpy",
                  "pandas"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nlq-y7f7nxdP"
      },
      "source": [
        "from gensim.models import KeyedVectors\n",
        "from gensim.test.utils import get_tmpfile\n",
        "from gensim.scripts.glove2word2vec import glove2word2vec\n",
        "glove2word2vec('Data/glove.42B.300d.txt', 'Data/glove.42B.300d_w2v.txt')\n",
        "word2vecmodel1 = KeyedVectors.load_word2vec_format('Data/glove.42B.300d_w2v.txt', binary=False)\n",
        "word2vecmodel1.save(\"Data/word2vec.model\")"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KrVyCVHdlWSc",
        "outputId": "dc5f4274-fca9-40f1-c3e6-8dfe903ad84d"
      },
      "source": [
        "import gc\n",
        "del word2vecmodel1\n",
        "gc.collect()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5HS3NNHzmX64"
      },
      "source": [
        "!rm Data/glove.42B.300d.txt\n",
        "!rm Data/glove.42B.300d_w2v.txt"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "53QcDc7AqkK5",
        "outputId": "6541b7fa-314a-4117-e92a-6310859d7c9e"
      },
      "source": [
        "from manual_training_inference import *"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Word statistics files not found!\n",
            "Downloading... done!\n",
            "Unpacking... done!\n",
            "Reading twitter - 1grams ...\n",
            "generating cache file for faster loading...\n",
            "reading ngrams /root/.ekphrasis/stats/twitter/counts_1grams.txt\n",
            "Reading twitter - 2grams ...\n",
            "generating cache file for faster loading...\n",
            "reading ngrams /root/.ekphrasis/stats/twitter/counts_2grams.txt\n",
            "Reading english - 1grams ...\n",
            "generating cache file for faster loading...\n",
            "reading ngrams /root/.ekphrasis/stats/english/counts_1grams.txt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MIKH2h5hzwcT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c9910f1f-6a71-42b1-88db-ffafb57c7a74"
      },
      "source": [
        "path_file='best_model_json/bestModel_birnnscrat.json'\n",
        "with open(path_file,mode='r') as f:\n",
        "    params = json.load(f)\n",
        "for key in params:\n",
        "    if params[key] == 'True':\n",
        "          params[key]=True\n",
        "    elif params[key] == 'False':\n",
        "          params[key]=False\n",
        "    if( key in ['batch_size','num_classes','hidden_size','supervised_layer_pos','num_supervised_heads','random_seed','max_length']):\n",
        "        if(params[key]!='N/A'):\n",
        "            params[key]=int(params[key])\n",
        "        \n",
        "    if((key == 'weights') and (params['auto_weights']==False)):\n",
        "        params[key] = ast.literal_eval(params[key])\n",
        "\n",
        "##### change in logging to output the results to neptune\n",
        "params['logging']='local'\n",
        "params['device']='cpu'\n",
        "params['best_params']=False\n",
        "\n",
        "if torch.cuda.is_available() and params['device']=='cuda':    \n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "else:\n",
        "    print('Since you dont want to use GPU, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n",
        "    \n",
        "    \n",
        "#### Few handy keys that you can directly change.\n",
        "params['variance']=1\n",
        "params['epochs']=5\n",
        "params['to_save']=True\n",
        "params['num_classes']=2\n",
        "params['data_file']=dict_data_folder[str(params['num_classes'])]['data_file']\n",
        "params['class_names']=dict_data_folder[str(params['num_classes'])]['class_label']\n",
        "if(params['num_classes']==2 and (params['auto_weights']==False)):\n",
        "      params['weights']=[1.0,1.0]\n",
        "        \n",
        "#for att_lambda in [0.001,0.01,0.1,1,10,100]\n",
        "train_model(params,device)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Since you dont want to use GPU, using the CPU instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  1%|          | 156/20148 [00:00<00:27, 726.27it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "total_data 20148\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 20148/20148 [00:27<00:00, 745.52it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "attention_error: 0\n",
            "no_majority: 919\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 10%|▉         | 1463/15383 [00:00<00:01, 7094.01it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "unk\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 15383/15383 [00:02<00:00, 7613.71it/s]\n",
            "  7%|▋         | 1077/15383 [00:00<00:01, 10769.80it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(22236, 300)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 15383/15383 [00:01<00:00, 11206.43it/s]\n",
            "100%|██████████| 1922/1922 [00:00<00:00, 11069.83it/s]\n",
            "100%|██████████| 1924/1924 [00:00<00:00, 10993.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "total dataset size: 19229\n",
            "[1.2301791 0.8423818]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:06,  2.58it/s]\n",
            "2it [00:00, 10.71it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.31485213757554\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [01:00,  7.97it/s]\n",
            "2it [00:00, 11.70it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.66\n",
            " Fscore: 0.65\n",
            " Precision: 0.74\n",
            " Recall: 0.70\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:01:01\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.42it/s]\n",
            "2it [00:00, 12.04it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.62\n",
            " Fscore: 0.62\n",
            " Precision: 0.71\n",
            " Recall: 0.67\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:06,  9.66it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.63\n",
            " Fscore: 0.62\n",
            " Precision: 0.73\n",
            " Recall: 0.68\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:06\n",
            "0.6163135612524757 0\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_2_100.pth\n",
            "\n",
            "======== Epoch 2 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:17,  2.44it/s]\n",
            "2it [00:00, 11.27it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.15798303045005\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:56,  8.44it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.73\n",
            " Fscore: 0.73\n",
            " Precision: 0.77\n",
            " Recall: 0.77\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:57\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.45it/s]\n",
            "2it [00:00, 11.10it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.67\n",
            " Fscore: 0.67\n",
            " Precision: 0.73\n",
            " Recall: 0.71\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.68\n",
            " Fscore: 0.68\n",
            " Precision: 0.73\n",
            " Recall: 0.71\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "0.6705669949950865 0.6163135612524757\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_2_100.pth\n",
            "\n",
            "======== Epoch 3 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [04:13,  1.90it/s]\n",
            "1it [00:00,  9.36it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.07526992512345\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [01:03,  7.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.81\n",
            " Fscore: 0.81\n",
            " Precision: 0.82\n",
            " Recall: 0.83\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:01:04\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 12.97it/s]\n",
            "2it [00:00, 11.88it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.71\n",
            " Fscore: 0.71\n",
            " Precision: 0.73\n",
            " Recall: 0.73\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:05\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:06,  9.72it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.72\n",
            " Fscore: 0.72\n",
            " Precision: 0.75\n",
            " Recall: 0.75\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:06\n",
            "0.7059588862867552 0.6705669949950865\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_2_100.pth\n",
            "\n",
            "======== Epoch 4 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [04:21,  1.84it/s]\n",
            "1it [00:00,  8.98it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.03198902027026\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [01:06,  7.25it/s]\n",
            "2it [00:00, 11.17it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.86\n",
            " Fscore: 0.86\n",
            " Precision: 0.85\n",
            " Recall: 0.87\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:01:07\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.59it/s]\n",
            "2it [00:00, 10.61it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.71\n",
            " Fscore: 0.71\n",
            " Precision: 0.72\n",
            " Recall: 0.73\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.42it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.71\n",
            " Fscore: 0.71\n",
            " Precision: 0.72\n",
            " Recall: 0.73\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "0.7129438316322797 0.7059588862867552\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_2_100.pth\n",
            "\n",
            "======== Epoch 5 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [04:00,  2.00it/s]\n",
            "1it [00:00,  9.25it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 294.9723350223533\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [01:06,  7.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.87\n",
            " Fscore: 0.87\n",
            " Precision: 0.87\n",
            " Recall: 0.88\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:01:07\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.44it/s]\n",
            "2it [00:00, 11.22it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.70\n",
            " Fscore: 0.70\n",
            " Precision: 0.71\n",
            " Recall: 0.72\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:07,  8.42it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.69\n",
            " Fscore: 0.69\n",
            " Precision: 0.70\n",
            " Recall: 0.71\n",
            " Roc Auc: 0.00\n",
            " Test took: 0:00:07\n",
            "best_val_fscore 0.7129438316322797\n",
            "best_test_fscore 0.7100710197860192\n",
            "best_val_rocauc 0\n",
            "best_test_rocauc 0\n",
            "best_val_precision 0.7192261279484664\n",
            "best_test_precision 0.7189857301769389\n",
            "best_val_recall 0.7266987311487441\n",
            "best_test_recall 0.7257290794182594\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ORBj47ArF8-F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a3e70d3-456f-4b7b-a526-3a12c42c05d7"
      },
      "source": [
        "params['num_classes']=3\n",
        "params['data_file']=dict_data_folder[str(params['num_classes'])]['data_file']\n",
        "params['class_names']=dict_data_folder[str(params['num_classes'])]['class_label']\n",
        "if(params['num_classes']==2 and (params['auto_weights']==False)):\n",
        "      params['weights']=[1.0,1.0]\n",
        "        \n",
        "#for att_lambda in [0.001,0.01,0.1,1,10,100]\n",
        "train_model(params,device)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1%|          | 122/20148 [00:00<00:38, 517.99it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "total_data 20148\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 20148/20148 [00:27<00:00, 722.31it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "attention_error: 0\n",
            "no_majority: 919\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  8%|▊         | 1298/15383 [00:00<00:02, 6588.67it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "unk\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 15383/15383 [00:02<00:00, 7279.06it/s]\n",
            "  7%|▋         | 1140/15383 [00:00<00:01, 11390.21it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(22236, 300)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 15383/15383 [00:01<00:00, 10952.63it/s]\n",
            "100%|██████████| 1922/1922 [00:00<00:00, 10938.67it/s]\n",
            "100%|██████████| 1924/1924 [00:00<00:00, 10802.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "total dataset size: 19229\n",
            "[1.0796857 0.8201194 1.1703163]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [02:53,  2.78it/s]\n",
            "2it [00:00, 13.38it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.6874583159068\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:33, 14.22it/s]\n",
            "2it [00:00, 13.70it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.62\n",
            " Fscore: 0.62\n",
            " Precision: 0.65\n",
            " Recall: 0.61\n",
            " Roc Auc: 0.80\n",
            " Test took: 0:00:34\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.37it/s]\n",
            "2it [00:00, 13.56it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.60\n",
            " Fscore: 0.59\n",
            " Precision: 0.64\n",
            " Recall: 0.58\n",
            " Roc Auc: 0.78\n",
            " Test took: 0:00:04\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.60it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.60\n",
            " Fscore: 0.59\n",
            " Precision: 0.63\n",
            " Recall: 0.58\n",
            " Roc Auc: 0.78\n",
            " Test took: 0:00:04\n",
            "0.5928381017318737 0\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "\n",
            "======== Epoch 2 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:03,  2.63it/s]\n",
            "2it [00:00, 14.48it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.51231660267916\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:33, 14.28it/s]\n",
            "2it [00:00, 14.34it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.69\n",
            " Fscore: 0.67\n",
            " Precision: 0.69\n",
            " Recall: 0.67\n",
            " Roc Auc: 0.84\n",
            " Test took: 0:00:34\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.40it/s]\n",
            "2it [00:00, 14.37it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.64\n",
            " Fscore: 0.62\n",
            " Precision: 0.64\n",
            " Recall: 0.62\n",
            " Roc Auc: 0.80\n",
            " Test took: 0:00:04\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.35it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.65\n",
            " Fscore: 0.62\n",
            " Precision: 0.64\n",
            " Recall: 0.62\n",
            " Roc Auc: 0.80\n",
            " Test took: 0:00:04\n",
            "0.6166377960748289 0.5928381017318737\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "\n",
            "======== Epoch 3 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:22,  2.38it/s]\n",
            "2it [00:00, 13.02it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.4182916906916\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:33, 14.25it/s]\n",
            "2it [00:00, 13.84it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.75\n",
            " Fscore: 0.74\n",
            " Precision: 0.74\n",
            " Recall: 0.74\n",
            " Roc Auc: 0.89\n",
            " Test took: 0:00:34\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.26it/s]\n",
            "2it [00:00, 14.61it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.64\n",
            " Fscore: 0.64\n",
            " Precision: 0.64\n",
            " Recall: 0.64\n",
            " Roc Auc: 0.81\n",
            " Test took: 0:00:04\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.30it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.65\n",
            " Fscore: 0.64\n",
            " Precision: 0.64\n",
            " Recall: 0.64\n",
            " Roc Auc: 0.82\n",
            " Test took: 0:00:04\n",
            "0.6365513941496782 0.6166377960748289\n",
            "Saving model\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "\n",
            "======== Epoch 4 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:30,  2.29it/s]\n",
            "2it [00:00, 13.86it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.3653224381984\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:33, 14.20it/s]\n",
            "2it [00:00, 13.80it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.76\n",
            " Fscore: 0.76\n",
            " Precision: 0.78\n",
            " Recall: 0.76\n",
            " Roc Auc: 0.92\n",
            " Test took: 0:00:34\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.21it/s]\n",
            "2it [00:00, 13.48it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.63\n",
            " Fscore: 0.63\n",
            " Precision: 0.66\n",
            " Recall: 0.62\n",
            " Roc Auc: 0.81\n",
            " Test took: 0:00:04\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.62\n",
            " Fscore: 0.62\n",
            " Precision: 0.65\n",
            " Recall: 0.61\n",
            " Roc Auc: 0.81\n",
            " Test took: 0:00:04\n",
            "\n",
            "======== Epoch 5 / 5 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [03:34,  2.24it/s]\n",
            "2it [00:00, 13.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "avg_train_loss 295.29857389644377\n",
            "model previously passed\n",
            "Running eval on  train ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "481it [00:34, 14.08it/s]\n",
            "2it [00:00, 13.85it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.81\n",
            " Fscore: 0.81\n",
            " Precision: 0.81\n",
            " Recall: 0.81\n",
            " Roc Auc: 0.93\n",
            " Test took: 0:00:35\n",
            "model previously passed\n",
            "Running eval on  val ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.03it/s]\n",
            "2it [00:00, 14.04it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.64\n",
            " Fscore: 0.63\n",
            " Precision: 0.63\n",
            " Recall: 0.63\n",
            " Roc Auc: 0.79\n",
            " Test took: 0:00:04\n",
            "model previously passed\n",
            "Running eval on  test ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "61it [00:04, 14.28it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Accuracy: 0.63\n",
            " Fscore: 0.62\n",
            " Precision: 0.62\n",
            " Recall: 0.62\n",
            " Roc Auc: 0.79\n",
            " Test took: 0:00:04\n",
            "best_val_fscore 0.6365513941496782\n",
            "best_test_fscore 0.6394014618815519\n",
            "best_val_rocauc 0.8117476454123843\n",
            "best_test_rocauc 0.8203029594853541\n",
            "best_val_precision 0.6370146992257849\n",
            "best_test_precision 0.6387270124237755\n",
            "best_val_recall 0.6361208687390083\n",
            "best_test_recall 0.6401887284784715\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRkvPokoMg3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88e0afdc-3386-47d4-fc04-05b35b3a68a6"
      },
      "source": [
        "import gc\n",
        "gc.collect()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "34jLhxdQ5stq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9fe326f4-3b0d-4608-e050-92212916c09d"
      },
      "source": [
        "!python testing_with_rational.py birnn_scrat 100\n",
        "!python testing_for_bias.py birnn_scrat 100"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-12-22 11:20:58.662290: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
            "/usr/local/lib/python3.6/dist-packages/spacy/util.py:275: UserWarning: [W031] Model 'en_core_web_sm' (2.2.5) requires spaCy v2.2 and is incompatible with the current spaCy version (2.3.2). This may lead to unexpected results or runtime errors. To resolve this, download a newer compatible model or retrain your custom model with the current spaCy version. For more details and available updates, run: python -m spacy validate\n",
            "  warnings.warn(warn_msg)\n",
            "Reading twitter - 1grams ...\n",
            "Reading twitter - 2grams ...\n",
            "Reading english - 1grams ...\n",
            "Using TensorFlow backend.\n",
            "Since you dont want to use GPU, using the CPU instead.\n",
            "tcmalloc: large alloc 2300993536 bytes == 0x41580000 @  0x7ff45ee3a1e7 0x59211c 0x4cddd0 0x5669e2 0x5a4cd1 0x4ddd76 0x5eae15 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "tcmalloc: large alloc 2300993536 bytes == 0xca7e6000 @  0x7ff45ee3a1e7 0x59211c 0x5eadd6 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:70: FutureWarning: Pass classes=['hatespeech' 'normal' 'offensive'], y=['normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'normal', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'offensive', 'hatespeech', 'offensive', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'normal', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'offensive', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'hatespeech', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'normal', 'hatespeech', 'normal', 'hatespeech', 'hatespeech', 'hatespeech', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal', 'offensive', 'hatespeech', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'offensive', 'offensive', 'normal', 'offensive', 'normal', 'hatespeech', 'normal', 'normal', 'offensive', 'hatespeech', 'normal', 'normal', 'hatespeech', 'offensive', 'normal', 'hatespeech', 'offensive', 'offensive', 'offensive', 'hatespeech', 'normal'] as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
            "  FutureWarning)\n",
            "total_data 1142\n",
            "100% 1142/1142 [00:01<00:00, 597.77it/s]\n",
            "100% 1142/1142 [00:00<00:00, 7759.15it/s]\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:54: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "Running eval on test data...\n",
            "100% 36/36 [00:02<00:00, 14.00it/s]\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            " Accuracy: 0.622\n",
            " Fscore: 0.457\n",
            " Precision: 0.520\n",
            " Recall: 0.411\n",
            " Test took: 0:00:03\n",
            "100% 1142/1142 [00:00<00:00, 3405.25it/s]\n",
            "Since you dont want to use GPU, using the CPU instead.\n",
            "tcmalloc: large alloc 2300993536 bytes == 0x46582000 @  0x7ff45ee3a1e7 0x59211c 0x4cddd0 0x5669e2 0x5a4cd1 0x4ddd76 0x5eae15 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "tcmalloc: large alloc 2300993536 bytes == 0xcf7e8000 @  0x7ff45ee3a1e7 0x59211c 0x5eadd6 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "100% 1142/1142 [00:00<00:00, 5799.05it/s]\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "Running eval on test data...\n",
            "100% 36/36 [00:02<00:00, 14.54it/s]\n",
            "100% 1142/1142 [00:00<00:00, 1528.77it/s]\n",
            "Since you dont want to use GPU, using the CPU instead.\n",
            "tcmalloc: large alloc 2300993536 bytes == 0x46582000 @  0x7ff45ee3a1e7 0x59211c 0x4cddd0 0x5669e2 0x5a4cd1 0x4ddd76 0x5eae15 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "tcmalloc: large alloc 2300993536 bytes == 0xcf7e8000 @  0x7ff45ee3a1e7 0x59211c 0x5eadd6 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7ff45ea37bf7 0x5b259a\n",
            "100% 1142/1142 [00:00<00:00, 5732.15it/s]\n",
            "Saved/birnnscrat_lstm_64_3_100.pth\n",
            "Running eval on test data...\n",
            "100% 36/36 [00:02<00:00, 14.58it/s]\n",
            "\u001b[0m2020-12-22 11:24:53.392536: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
            "/usr/local/lib/python3.6/dist-packages/spacy/util.py:275: UserWarning: [W031] Model 'en_core_web_sm' (2.2.5) requires spaCy v2.2 and is incompatible with the current spaCy version (2.3.2). This may lead to unexpected results or runtime errors. To resolve this, download a newer compatible model or retrain your custom model with the current spaCy version. For more details and available updates, run: python -m spacy validate\n",
            "  warnings.warn(warn_msg)\n",
            "Reading twitter - 1grams ...\n",
            "Reading twitter - 2grams ...\n",
            "Reading english - 1grams ...\n",
            "Using TensorFlow backend.\n",
            "tcmalloc: large alloc 2300993536 bytes == 0x40516000 @  0x7f03c97c31e7 0x59211c 0x4cddd0 0x5669e2 0x5a4cd1 0x4ddd76 0x5eae15 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7f03c93c0bf7 0x5b259a\n",
            "tcmalloc: large alloc 2300993536 bytes == 0xc977c000 @  0x7f03c97c31e7 0x59211c 0x5eadd6 0x4debf7 0x5ebace 0x50a25a 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x50ad03 0x634e72 0x634f27 0x6386df 0x639281 0x4b0dc0 0x7f03c93c0bf7 0x5b259a\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:70: FutureWarning: Pass classes=['non-toxic' 'toxic'], y=['non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'non-toxic', 'toxic', 'toxic', 'non-toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'toxic', 'non-toxic'] as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
            "  FutureWarning)\n",
            "total_data 1924\n",
            "100% 1924/1924 [00:02<00:00, 693.19it/s]\n",
            "100% 1924/1924 [00:00<00:00, 7806.50it/s]\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:54: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n",
            "Saved/birnnscrat_lstm_64_2_100.pth\n",
            "Running eval on test data...\n",
            "100% 61/61 [00:04<00:00, 15.15it/s]\n",
            "\u001b[0m"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SynOxIW5PY-v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1beaf1d-206f-4d4c-b562-a1d3bd404f45"
      },
      "source": [
        "!ls explanations_dicts"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "bestModel_birnnscrat_100_explanation_top5.json\tbestModel_birnnscrat_bias.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7DVEb9O3IlCd"
      },
      "source": [
        ""
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A9Ka6ukjTC5M"
      },
      "source": [
        ""
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r1-wjKhvNrF5"
      },
      "source": [
        "\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "**Bias Calculation**\r\n",
        "\r\n",
        "---\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GAPagtSDQ9zo"
      },
      "source": [
        "from collections import Counter,defaultdict\r\n",
        "from tqdm.notebook import tqdm\r\n",
        "import json\r\n",
        "import numpy as np"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFOmyTVIRJ7R"
      },
      "source": [
        "# get_annotated_data method is used to load the dataset\r\n",
        "from Preprocess.dataCollect import get_annotated_data"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iw2qaVXLROW4"
      },
      "source": [
        "dict_data_folder={\r\n",
        "      '2':{'data_file':'Data/dataset.json','class_label':'Data/classes_two.npy'},\r\n",
        "      '3':{'data_file':'Data/dataset.json','class_label':'Data/classes.npy'}\r\n",
        "}\r\n",
        "\r\n",
        "params = {}\r\n",
        "\r\n",
        "# We need to load the dataset with the labels as 'toxic' and 'non-toxic'. \r\n",
        "# We consider hatespeech and offensive as toxic and normal as non-toxic.\r\n",
        "params['num_classes']=2  \r\n",
        "params['data_file']=dict_data_folder[str(params['num_classes'])]['data_file']\r\n",
        "params['class_names']=dict_data_folder[str(params['num_classes'])]['class_label']\r\n",
        "\r\n",
        "data_all_labelled=get_annotated_data(params)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LDRrrN8CRRfg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 580
        },
        "outputId": "d2c46d83-ef42-4c3e-fa60-5f3adc1f301a"
      },
      "source": [
        "data_all_labelled"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th style=\"min-width: {};\"></th>\n",
              "      <th style=\"min-width: {};\">post_id</th>\n",
              "      <th style=\"min-width: {};\">text</th>\n",
              "      <th style=\"min-width: {};\">annotatorid1</th>\n",
              "      <th style=\"min-width: {};\">target1</th>\n",
              "      <th style=\"min-width: {};\">label1</th>\n",
              "      <th style=\"min-width: {};\">annotatorid2</th>\n",
              "      <th style=\"min-width: {};\">target2</th>\n",
              "      <th style=\"min-width: {};\">label2</th>\n",
              "      <th style=\"min-width: {};\">annotatorid3</th>\n",
              "      <th style=\"min-width: {};\">target3</th>\n",
              "      <th style=\"min-width: {};\">label3</th>\n",
              "      <th style=\"min-width: {};\">rationales</th>\n",
              "      <th style=\"min-width: {};\">final_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1179055004553900032_twitter</td>\n",
              "      <td>[i, dont, think, im, getting, my, baby, them, ...</td>\n",
              "      <td>1</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>2</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>3</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1179063826874032128_twitter</td>\n",
              "      <td>[we, cannot, continue, calling, ourselves, fem...</td>\n",
              "      <td>1</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>2</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>3</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1178793830532956161_twitter</td>\n",
              "      <td>[nawt, yall, niggers, ignoring, me]</td>\n",
              "      <td>4</td>\n",
              "      <td>[African]</td>\n",
              "      <td>normal</td>\n",
              "      <td>2</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>3</td>\n",
              "      <td>[African]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1179088797964763136_twitter</td>\n",
              "      <td>[&lt;user&gt;, i, am, bit, confused, coz, chinese, p...</td>\n",
              "      <td>1</td>\n",
              "      <td>[Asian]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>4</td>\n",
              "      <td>[Asian]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>3</td>\n",
              "      <td>[Asian]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1179085312976445440_twitter</td>\n",
              "      <td>[this, bitch, in, whataburger, eating, a, burg...</td>\n",
              "      <td>4</td>\n",
              "      <td>[Caucasian, Women]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>2</td>\n",
              "      <td>[Women, Caucasian]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>3</td>\n",
              "      <td>[Women, Caucasian]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20143</th>\n",
              "      <td>9989999_gab</td>\n",
              "      <td>[if, ur, still, on, twitter, tell, carlton, i,...</td>\n",
              "      <td>217</td>\n",
              "      <td>[Men, Women, Other]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>199</td>\n",
              "      <td>[None]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>215</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20144</th>\n",
              "      <td>9990225_gab</td>\n",
              "      <td>[when, i, first, got, on, here, and, said, i, ...</td>\n",
              "      <td>220</td>\n",
              "      <td>[African]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>223</td>\n",
              "      <td>[African, Other]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>231</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20145</th>\n",
              "      <td>9991681_gab</td>\n",
              "      <td>[was, macht, der, moslem, wenn, der, zion, geg...</td>\n",
              "      <td>206</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>203</td>\n",
              "      <td>[Other]</td>\n",
              "      <td>normal</td>\n",
              "      <td>211</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20146</th>\n",
              "      <td>9992513_gab</td>\n",
              "      <td>[it, is, awful, look, at, world, demographics,...</td>\n",
              "      <td>209</td>\n",
              "      <td>[Hispanic]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>253</td>\n",
              "      <td>[Asian]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>222</td>\n",
              "      <td>[Asian]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20147</th>\n",
              "      <td>9998729_gab</td>\n",
              "      <td>[the, jewish, globalist, elite, have, only, im...</td>\n",
              "      <td>200</td>\n",
              "      <td>[African, Islam]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>202</td>\n",
              "      <td>[Islam, Jewish]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>207</td>\n",
              "      <td>[African, Islam, Jewish]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>20148 rows × 13 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                           post_id  ... final_label\n",
              "0      1179055004553900032_twitter  ...   non-toxic\n",
              "1      1179063826874032128_twitter  ...   non-toxic\n",
              "2      1178793830532956161_twitter  ...   non-toxic\n",
              "3      1179088797964763136_twitter  ...       toxic\n",
              "4      1179085312976445440_twitter  ...       toxic\n",
              "...                            ...  ...         ...\n",
              "20143                  9989999_gab  ...       toxic\n",
              "20144                  9990225_gab  ...       toxic\n",
              "20145                  9991681_gab  ...   non-toxic\n",
              "20146                  9992513_gab  ...       toxic\n",
              "20147                  9998729_gab  ...       toxic\n",
              "\n",
              "[20148 rows x 13 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14K9-nzFRU8B"
      },
      "source": [
        "def generate_target_information(dataset):\r\n",
        "    final_target_output = defaultdict(list)\r\n",
        "    all_communities_selected = []\r\n",
        "    \r\n",
        "    for each in dataset.iterrows(): \r\n",
        "        # All the target communities tagged for this post\r\n",
        "        all_targets = each[1]['target1']+each[1]['target2']+each[1]['target3']  \r\n",
        "        community_dict = dict(Counter(all_targets))\r\n",
        "        \r\n",
        "        # Select only those communities which are present more than once.\r\n",
        "        for key in community_dict:\r\n",
        "            if community_dict[key]>1:  \r\n",
        "                final_target_output[each[1]['post_id']].append(key)\r\n",
        "                all_communities_selected.append(key)\r\n",
        "        \r\n",
        "        # If no community is selected based on majority voting then we don't select any community\r\n",
        "        if each[1]['post_id'] not in final_target_output:\r\n",
        "            final_target_output[each[1]['post_id']].append('None')\r\n",
        "            all_communities_selected.append(key)\r\n",
        "\r\n",
        "    return final_target_output, all_communities_selected"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mEmm7AD9Ra13"
      },
      "source": [
        "target_information, all_communities_selected = generate_target_information(data_all_labelled)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LPCh_pj3ReFA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af84806a-7064-4bf5-d1b1-ff26f1fda0fc"
      },
      "source": [
        "community_count_dict = Counter(all_communities_selected)\r\n",
        "\r\n",
        "# We remove None and Other from dictionary\r\n",
        "community_count_dict.pop('None')\r\n",
        "community_count_dict.pop('Other')\r\n",
        "\r\n",
        "# For the bias calculation, we are considering the top 10 communites based on their count\r\n",
        "list_selected_community = [community for community, value in community_count_dict.most_common(10)]\r\n",
        "list_selected_community"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['African',\n",
              " 'Islam',\n",
              " 'Jewish',\n",
              " 'Homosexual',\n",
              " 'Women',\n",
              " 'Refugee',\n",
              " 'Arab',\n",
              " 'Caucasian',\n",
              " 'Asian',\n",
              " 'Hispanic']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRU3peSrRhVo"
      },
      "source": [
        "# Based on the top 10 communities, we filter the target_information\r\n",
        "# This will remove the other communities from the calculation\r\n",
        "\r\n",
        "final_target_information ={}\r\n",
        "for each in target_information:\r\n",
        "    temp = list(set(target_information[each])&set(list_selected_community))\r\n",
        "    if len(temp) == 0:\r\n",
        "        final_target_information[each] = None\r\n",
        "    else:\r\n",
        "        final_target_information[each] = temp"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BEsvl0xaRkpw"
      },
      "source": [
        "# Add a new column 'final_target_category' which will contain the selected target community names\r\n",
        "data_all_labelled['final_target_category'] = data_all_labelled['post_id'].map(final_target_information)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0-Y6iMiRnxH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "outputId": "a701921e-ffb8-459d-ab3f-70c2b7636991"
      },
      "source": [
        "# The post_id_divisions file stores the train, val, test split ids. We select only the test ids.\r\n",
        "postpost_id_divisions_path = './Data/post_id_divisions.json'\r\n",
        "\r\n",
        "with open(postpost_id_divisions_path, 'r') as fp:\r\n",
        "    post_id_dict=json.load(fp)\r\n",
        "\r\n",
        "data_all_labelled_bias = data_all_labelled[data_all_labelled['post_id'].isin(post_id_dict['test'])]\r\n",
        "data_all_labelled_bias"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th style=\"min-width: {};\"></th>\n",
              "      <th style=\"min-width: {};\">post_id</th>\n",
              "      <th style=\"min-width: {};\">text</th>\n",
              "      <th style=\"min-width: {};\">annotatorid1</th>\n",
              "      <th style=\"min-width: {};\">target1</th>\n",
              "      <th style=\"min-width: {};\">label1</th>\n",
              "      <th style=\"min-width: {};\">annotatorid2</th>\n",
              "      <th style=\"min-width: {};\">target2</th>\n",
              "      <th style=\"min-width: {};\">label2</th>\n",
              "      <th style=\"min-width: {};\">annotatorid3</th>\n",
              "      <th style=\"min-width: {};\">target3</th>\n",
              "      <th style=\"min-width: {};\">label3</th>\n",
              "      <th style=\"min-width: {};\">rationales</th>\n",
              "      <th style=\"min-width: {};\">final_label</th>\n",
              "      <th style=\"min-width: {};\">final_target_category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1179055004553900032_twitter</td>\n",
              "      <td>[i, dont, think, im, getting, my, baby, them, ...</td>\n",
              "      <td>1</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>2</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>3</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>13851720_gab</td>\n",
              "      <td>[laura, loomer, raped, me, while, screaming, a...</td>\n",
              "      <td>1</td>\n",
              "      <td>[Jewish]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>2</td>\n",
              "      <td>[Jewish]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>3</td>\n",
              "      <td>[Jewish]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Jewish]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>1178818409812746240_twitter</td>\n",
              "      <td>[&lt;user&gt;, what, did, the, old, lady, do, was, s...</td>\n",
              "      <td>9</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>10</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>4</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>19346774_gab</td>\n",
              "      <td>[as, much, as, i, appreciate, a, format, of, o...</td>\n",
              "      <td>9</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>13</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>4</td>\n",
              "      <td>[Hispanic]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>1165819983701643266_twitter</td>\n",
              "      <td>[sex, be, so, good, a, bitch, be, slow, stroki...</td>\n",
              "      <td>4</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>7</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>16</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], [0, 0, 0, ...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Women]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20105</th>\n",
              "      <td>9773208_gab</td>\n",
              "      <td>[it, always, women, trying, this, shit, like, ...</td>\n",
              "      <td>200</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>202</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>203</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Women]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20109</th>\n",
              "      <td>9802943_gab</td>\n",
              "      <td>[because, women, would, never, lie, about, bei...</td>\n",
              "      <td>228</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>222</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>209</td>\n",
              "      <td>[Women]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Women]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20117</th>\n",
              "      <td>9826963_gab</td>\n",
              "      <td>[how, is, all, that, awesome, muslim, diversit...</td>\n",
              "      <td>228</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>222</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>209</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Islam]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20128</th>\n",
              "      <td>9867117_gab</td>\n",
              "      <td>[well, my, dear, lgbtq, brothers, and, sisters...</td>\n",
              "      <td>207</td>\n",
              "      <td>[Islam, Other]</td>\n",
              "      <td>offensive</td>\n",
              "      <td>223</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>231</td>\n",
              "      <td>[Islam]</td>\n",
              "      <td>hatespeech</td>\n",
              "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
              "      <td>toxic</td>\n",
              "      <td>[Islam]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20130</th>\n",
              "      <td>9869181_gab</td>\n",
              "      <td>[but, it, on, hbo, more, violence, and, sex, c...</td>\n",
              "      <td>223</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>200</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>233</td>\n",
              "      <td>[None]</td>\n",
              "      <td>normal</td>\n",
              "      <td>[]</td>\n",
              "      <td>non-toxic</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1924 rows × 14 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                           post_id  ... final_target_category\n",
              "0      1179055004553900032_twitter  ...                  None\n",
              "5                     13851720_gab  ...              [Jewish]\n",
              "17     1178818409812746240_twitter  ...                  None\n",
              "25                    19346774_gab  ...                  None\n",
              "36     1165819983701643266_twitter  ...               [Women]\n",
              "...                            ...  ...                   ...\n",
              "20105                  9773208_gab  ...               [Women]\n",
              "20109                  9802943_gab  ...               [Women]\n",
              "20117                  9826963_gab  ...               [Islam]\n",
              "20128                  9867117_gab  ...               [Islam]\n",
              "20130                  9869181_gab  ...                  None\n",
              "\n",
              "[1924 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uT3uzWgcRqaf"
      },
      "source": [
        "from sklearn.metrics import roc_auc_score\r\n",
        "\r\n",
        "# The names of the files which will contain the bias output for each model\r\n",
        "bias_score_file_mapping={\r\n",
        "    'BiRNN-Attn':'bestModel_birnnscrat_bias.json',\r\n",
        "}\r\n",
        "\r\n",
        "# The parent folder path of the bias output files\r\n",
        "parent_path = './explanations_dicts/'\r\n",
        "\r\n",
        "# The bias methods that will be considered\r\n",
        "method_list = ['subgroup', 'bpsn', 'bnsp']\r\n",
        "\r\n",
        "community_list = list(list_selected_community)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X0H9nMZNR1v3"
      },
      "source": [
        "# This function is used to convert the classification into a [0-1] score \r\n",
        "# with a value of 0 meaning non-toxic and 1 meaning toxic\r\n",
        "def convert_to_score(label_name, label_dict):\r\n",
        "    if label_name=='non-toxic':\r\n",
        "        return 1-label_dict[label_name]\r\n",
        "    else:\r\n",
        "        return label_dict[label_name]"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qUFyY8iVR4kn"
      },
      "source": [
        "# Function to divide the ids into postive or class class based on the method.\r\n",
        "def bias_evaluation_metric(dataset, method, community):\r\n",
        "    positive_ids = []\r\n",
        "    negative_ids = []\r\n",
        "    if method=='subgroup':\r\n",
        "        for eachrow in dataset.iterrows():\r\n",
        "            if eachrow[1]['final_target_category'] == None:\r\n",
        "                continue\r\n",
        "            if community in eachrow[1]['final_target_category']:\r\n",
        "                if eachrow[1]['final_label'] =='non-toxic':\r\n",
        "                    negative_ids.append(eachrow[1]['post_id'])\r\n",
        "                else:\r\n",
        "                    positive_ids.append(eachrow[1]['post_id'])\r\n",
        "            else:\r\n",
        "                pass\r\n",
        "    elif method=='bpsn':\r\n",
        "        for eachrow in dataset.iterrows():\r\n",
        "            if eachrow[1]['final_target_category'] == None:\r\n",
        "                continue\r\n",
        "            if community in eachrow[1]['final_target_category']:\r\n",
        "#                 print(eachrow[1]['final_label'])\r\n",
        "                if eachrow[1]['final_label'] =='non-toxic':\r\n",
        "                    negative_ids.append(eachrow[1]['post_id'])\r\n",
        "                else:\r\n",
        "                    pass\r\n",
        "            else:\r\n",
        "                if eachrow[1]['final_label'] !='non-toxic':\r\n",
        "                    positive_ids.append(eachrow[1]['post_id'])\r\n",
        "                else:\r\n",
        "                    pass\r\n",
        "    elif method=='bnsp':\r\n",
        "        for eachrow in dataset.iterrows():\r\n",
        "            if eachrow[1]['final_target_category'] == None:\r\n",
        "                continue\r\n",
        "            if community in eachrow[1]['final_target_category']:\r\n",
        "                if eachrow[1]['final_label'] !='non-toxic':\r\n",
        "                    positive_ids.append(eachrow[1]['post_id'])\r\n",
        "                else:\r\n",
        "                    pass\r\n",
        "            else:\r\n",
        "                if eachrow[1]['final_label'] =='non-toxic':\r\n",
        "                    negative_ids.append(eachrow[1]['post_id'])\r\n",
        "                else:\r\n",
        "                    pass\r\n",
        "    else:\r\n",
        "        print('Incorrect option selected!!!')\r\n",
        "                \r\n",
        "    return {'positiveID':positive_ids, 'negativeID':negative_ids}"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_o-CBxRFR7YQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "6d248353b8b04ebcaab705f0629a4438",
            "6bffe5fef8d74bbaa31a61c0f1a32020",
            "936b8a4513ac449ab0c77c2f912f1960",
            "004b043ce4dc436ab1d8ee4f9fb6873b",
            "89755305bd01495b8f54d6184d3cd844",
            "01f24a559b8741aaa137adc5729a75a4",
            "2c1575998e31458e811646dcaba8c759",
            "091ff09479a74529a425affa22a500be"
          ]
        },
        "outputId": "d7630d52-0855-4178-d4a6-e839443cccd4"
      },
      "source": [
        "final_bias_dictionary = defaultdict(lambda: defaultdict(dict))\r\n",
        "\r\n",
        "# We load each of the model bias output file and compute the bias score using each method for all the community\r\n",
        "for each_model in tqdm(bias_score_file_mapping):\r\n",
        "    total_data ={}\r\n",
        "    with open(parent_path+bias_score_file_mapping[each_model]) as fp:\r\n",
        "        for line in fp:\r\n",
        "            data = json.loads(line)\r\n",
        "            total_data[data['annotation_id']] = data\r\n",
        "    for each_method in method_list:\r\n",
        "        for each_community in community_list:\r\n",
        "            community_data = bias_evaluation_metric(data_all_labelled_bias, each_method, each_community)\r\n",
        "            truth_values = []\r\n",
        "            prediction_values = []\r\n",
        "\r\n",
        "\r\n",
        "            label_to_value = {'toxic':1.0, 'non-toxic':0.0}\r\n",
        "            for each in community_data['positiveID']:\r\n",
        "                truth_values.append(label_to_value[total_data[each]['ground_truth']])\r\n",
        "                prediction_values.append(convert_to_score(total_data[each]['classification'], total_data[each]['classification_scores']))\r\n",
        "\r\n",
        "            for each in community_data['negativeID']:\r\n",
        "                truth_values.append(label_to_value[total_data[each]['ground_truth']])\r\n",
        "                prediction_values.append(convert_to_score(total_data[each]['classification'], total_data[each]['classification_scores']))\r\n",
        "\r\n",
        "            roc_output_value = roc_auc_score(truth_values, prediction_values)\r\n",
        "            final_bias_dictionary[each_model][each_method][each_community] = roc_output_value"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6d248353b8b04ebcaab705f0629a4438",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_uoz4r8JR-23",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "166586a8-966c-4389-e863-28cb011b7bea"
      },
      "source": [
        "%precision 4"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'%.4f'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIQi-Am9SCIz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2f7254d-5fb6-4a27-a315-5b00724a68c2"
      },
      "source": [
        "# To combine the per-identity Bias AUCs into one overall measure, we calculate their generalized mean as defined below:\r\n",
        "power_value = -5\r\n",
        "num_communities = len(community_list)\r\n",
        "\r\n",
        "for each_model in final_bias_dictionary:\r\n",
        "    for each_method in final_bias_dictionary[each_model]:\r\n",
        "        temp_value =[]\r\n",
        "        for each_community in final_bias_dictionary[each_model][each_method]:\r\n",
        "            temp_value.append(pow(final_bias_dictionary[each_model][each_method][each_community], power_value))\r\n",
        "        print(each_model, each_method, pow(np.sum(temp_value)/num_communities, 1/power_value))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BiRNN-Attn subgroup 0.6641103137568249\n",
            "BiRNN-Attn bpsn 0.6564215397561137\n",
            "BiRNN-Attn bnsp 0.7040256983333031\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jx1-xKrKSFza"
      },
      "source": [
        ""
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cos2FyRyScI6"
      },
      "source": [
        "\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "\r\n",
        "**Calculate Explainability**\r\n",
        "\r\n",
        "---\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0n04ccES0G3"
      },
      "source": [
        "import json\r\n",
        "from tqdm.notebook import tqdm\r\n",
        "import more_itertools as mit\r\n",
        "import os"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQ37OLVZS8gB"
      },
      "source": [
        "# get_annotated_data method is used to load the dataset\r\n",
        "from Preprocess import *\r\n",
        "from Preprocess.dataCollect import *"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8AZCJG3wS-ko"
      },
      "source": [
        "dict_data_folder={\r\n",
        "      '2':{'data_file':'Data/dataset.json','class_label':'Data/classes_two.npy'},\r\n",
        "      '3':{'data_file':'Data/dataset.json','class_label':'Data/classes.npy'}\r\n",
        "}\r\n",
        "\r\n",
        "# We need to load the dataset with the labels as 'hatespeech', 'offensive', and 'normal' (3-class). \r\n",
        "\r\n",
        "params = {}\r\n",
        "params['num_classes']=3\r\n",
        "params['data_file']=dict_data_folder[str(params['num_classes'])]['data_file']\r\n",
        "params['class_names']=dict_data_folder[str(params['num_classes'])]['class_label']\r\n",
        "\r\n",
        "data_all_labelled=get_annotated_data(params)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H5DLppxPTAjo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b522fae-df8c-422e-e17a-465203fd67da"
      },
      "source": [
        "# The important key here is the 'bert_token'. Set it to True for Bert based models and False for Others.\r\n",
        "\r\n",
        "params_data={\r\n",
        "    'include_special':False,  #True is want to include <url> in place of urls if False will be removed\r\n",
        "    'bert_tokens':False, #True /False\r\n",
        "    'type_attention':'softmax', #softmax\r\n",
        "    'set_decay':0.1,\r\n",
        "    'majority':2,\r\n",
        "    'max_length':128,\r\n",
        "    'variance':10,\r\n",
        "    'window':4,\r\n",
        "    'alpha':0.5,\r\n",
        "    'p_value':0.8,\r\n",
        "    'method':'additive',\r\n",
        "    'decay':False,\r\n",
        "    'normalized':False,\r\n",
        "    'not_recollect':True,\r\n",
        "}\r\n",
        "\r\n",
        "\r\n",
        "if(params_data['bert_tokens']):\r\n",
        "    print('Loading BERT tokenizer...')\r\n",
        "    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=False)\r\n",
        "else:\r\n",
        "    print('Loading Normal tokenizer...')\r\n",
        "    tokenizer=None"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading Normal tokenizer...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Aqwkmy9ITEvH"
      },
      "source": [
        "# Load the whole dataset and get the tokenwise rationales\r\n",
        "def get_training_data(data):\r\n",
        "    post_ids_list=[]\r\n",
        "    text_list=[]\r\n",
        "    attention_list=[]\r\n",
        "    label_list=[]\r\n",
        "    \r\n",
        "    final_binny_output = []\r\n",
        "    print('total_data',len(data))\r\n",
        "    for index,row in tqdm(data.iterrows(),total=len(data)):\r\n",
        "        annotation=row['final_label']\r\n",
        "        \r\n",
        "        text=row['text']\r\n",
        "        post_id=row['post_id']\r\n",
        "        annotation_list=[row['label1'],row['label2'],row['label3']]\r\n",
        "        tokens_all = list(row['text'])\r\n",
        "#         attention_masks =  [list(row['explain1']),list(row['explain2']),list(row['explain1'])]\r\n",
        "        \r\n",
        "        if(annotation!= 'undecided'):\r\n",
        "            tokens_all,attention_masks=returnMask(row, params_data, tokenizer)\r\n",
        "            final_binny_output.append([post_id, annotation, tokens_all, attention_masks, annotation_list])\r\n",
        "\r\n",
        "    return final_binny_output"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2lkIo1ATHjH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "932cadbf-42a0-49a2-ac9c-270746395313"
      },
      "source": [
        "training_data=get_training_data(data_all_labelled)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 69/20148 [00:00<00:29, 673.57it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "total_data 20148\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 20148/20148 [00:26<00:00, 757.98it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZxfQUvdTJzn"
      },
      "source": [
        "# https://stackoverflow.com/questions/2154249/identify-groups-of-continuous-numbers-in-a-list\r\n",
        "def find_ranges(iterable):\r\n",
        "    \"\"\"Yield range of consecutive numbers.\"\"\"\r\n",
        "    for group in mit.consecutive_groups(iterable):\r\n",
        "        group = list(group)\r\n",
        "        if len(group) == 1:\r\n",
        "            yield group[0]\r\n",
        "        else:\r\n",
        "            yield group[0], group[-1]\r\n",
        "            \r\n",
        "# Convert dataset into ERASER format: https://github.com/jayded/eraserbenchmark/blob/master/rationale_benchmark/utils.py\r\n",
        "def get_evidence(post_id, anno_text, explanations):\r\n",
        "    output = []\r\n",
        "\r\n",
        "    indexes = sorted([i for i, each in enumerate(explanations) if each==1])\r\n",
        "    span_list = list(find_ranges(indexes))\r\n",
        "\r\n",
        "    for each in span_list:\r\n",
        "        if type(each)== int:\r\n",
        "            start = each\r\n",
        "            end = each+1\r\n",
        "        elif len(each) == 2:\r\n",
        "            start = each[0]\r\n",
        "            end = each[1]+1\r\n",
        "        else:\r\n",
        "            print('error')\r\n",
        "\r\n",
        "        output.append({\"docid\":post_id, \r\n",
        "              \"end_sentence\": -1, \r\n",
        "              \"end_token\": end, \r\n",
        "              \"start_sentence\": -1, \r\n",
        "              \"start_token\": start, \r\n",
        "              \"text\": ' '.join([str(x) for x in anno_text[start:end]])})\r\n",
        "    return output\r\n",
        "\r\n",
        "# To use the metrices defined in ERASER, we will have to convert the dataset\r\n",
        "def convert_to_eraser_format(dataset, method, save_split, save_path, id_division):  \r\n",
        "    final_output = []\r\n",
        "    \r\n",
        "    if save_split:\r\n",
        "        train_fp = open(save_path+'train.jsonl', 'w')\r\n",
        "        val_fp = open(save_path+'val.jsonl', 'w')\r\n",
        "        test_fp = open(save_path+'test.jsonl', 'w')\r\n",
        "            \r\n",
        "    for tcount, eachrow in enumerate(dataset):\r\n",
        "        \r\n",
        "        temp = {}\r\n",
        "        post_id = eachrow[0]\r\n",
        "        post_class = eachrow[1]\r\n",
        "        anno_text_list = eachrow[2]\r\n",
        "        majority_label = eachrow[1]\r\n",
        "        \r\n",
        "        if majority_label=='normal':\r\n",
        "            continue\r\n",
        "        \r\n",
        "        all_labels = eachrow[4]\r\n",
        "        explanations = []\r\n",
        "        for each_explain in eachrow[3]:\r\n",
        "            explanations.append(list(each_explain))\r\n",
        "        \r\n",
        "        # For this work, we have considered the union of explanations. Other options could be explored as well.\r\n",
        "        if method == 'union':\r\n",
        "            final_explanation = [any(each) for each in zip(*explanations)]\r\n",
        "            final_explanation = [int(each) for each in final_explanation]\r\n",
        "        \r\n",
        "            \r\n",
        "        temp['annotation_id'] = post_id\r\n",
        "        temp['classification'] = post_class\r\n",
        "        temp['evidences'] = [get_evidence(post_id, list(anno_text_list), final_explanation)]\r\n",
        "        temp['query'] = \"What is the class?\"\r\n",
        "        temp['query_type'] = None\r\n",
        "        final_output.append(temp)\r\n",
        "        \r\n",
        "        if save_split:\r\n",
        "            if not os.path.exists(save_path+'docs'):\r\n",
        "                os.makedirs(save_path+'docs')\r\n",
        "            \r\n",
        "            with open(save_path+'docs/'+post_id, 'w') as fp:\r\n",
        "                fp.write(' '.join([str(x) for x in list(anno_text_list)]))\r\n",
        "            \r\n",
        "            if post_id in id_division['train']:\r\n",
        "                train_fp.write(json.dumps(temp)+'\\n')\r\n",
        "            \r\n",
        "            elif post_id in id_division['val']:\r\n",
        "                val_fp.write(json.dumps(temp)+'\\n')\r\n",
        "            \r\n",
        "            elif post_id in id_division['test']:\r\n",
        "                test_fp.write(json.dumps(temp)+'\\n')\r\n",
        "            else:\r\n",
        "                print(post_id)\r\n",
        "    \r\n",
        "    if save_split:\r\n",
        "        train_fp.close()\r\n",
        "        val_fp.close()\r\n",
        "        test_fp.close()\r\n",
        "        \r\n",
        "    return final_output"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "deYKnU3wTRJn"
      },
      "source": [
        "# The post_id_divisions file stores the train, val, test split ids. We select only the test ids.\r\n",
        "with open('./Data/post_id_divisions.json') as fp:\r\n",
        "    id_division = json.load(fp)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlA0iMeETjUd"
      },
      "source": [
        "!mkdir ./Data/Evaluation\r\n",
        "!mkdir ./Data/Evaluation/Model_Eval"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2XwjEPOTUaY"
      },
      "source": [
        "method = 'union'\r\n",
        "save_split = True\r\n",
        "save_path = './Data/Evaluation/Model_Eval/'  #The dataset in Eraser Format will be stored here.\r\n",
        "output_eraser = convert_to_eraser_format(training_data, method, save_split, save_path, id_division)"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e76FcTICTXrX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0a59700-2351-408d-e45f-f7f64823b673"
      },
      "source": [
        "!ls Data/Evaluation/Model_Eval/"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "docs  test.jsonl  train.jsonl  val.jsonl\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9lpwdAeTaf_"
      },
      "source": [
        ""
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kMWLQB2uT28g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0b33f6d3-58d4-4d1a-8ae7-1bc901eab632"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best_model_json\t\t\t     Models\n",
            "best_runs.sh\t\t\t     Parameters_description.md\n",
            "Bias_Calculation_NB.ipynb\t     parameters_selection.py\n",
            "convert_to_word2vec.py\t\t     Preprocess\n",
            "Data\t\t\t\t     __pycache__\n",
            "eraserbenchmark\t\t\t     README.md\n",
            "Example_HateExplain.ipynb\t     requirements.txt\n",
            "Explainability_Calculation_NB.ipynb  Saved\n",
            "explanations_dicts\t\t     TensorDataset\n",
            "Figures\t\t\t\t     testing_for_bias.py\n",
            "LICENSE\t\t\t\t     testing_with_lime.py\n",
            "manual_training_inference.py\t     testing_with_rational.py\n",
            "model_explain_output.json\t     test_parallel.sh\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ylHvxsmoUv7Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "82499480-1934-4fb1-a70b-35fa1626222a"
      },
      "source": [
        "cd eraserbenchmark/"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/HateXplain/eraserbenchmark\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SajAK6cMUyt6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecdd495c-e6e7-41a1-e5e6-c419783aa9ae"
      },
      "source": [
        "!ls "
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data_exploration.ipynb\tparams\t\t     README.md\t       requirements.txt\n",
            "LICENSE\t\t\trationale_benchmark  REPRODUCTION.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0iudyL6lcXib"
      },
      "source": [
        ""
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzZLhJf-U8Vf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0eda6e6-2b4d-44b5-8485-c720441940f0"
      },
      "source": [
        "!PYTHONPATH=./:$PYTHONPATH python rationale_benchmark/metrics.py --split test  --data_dir ../Data/Evaluation/Model_Eval --results ../explanations_dicts/bestModel_birnnscrat_100_explanation_top5.json --score_file ../model_explain_output.json"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "   967 MainThread Error in instances: 0 instances fail validation: set()\n",
            "  2453 MainThread No sentence level predictions detected, skipping sentence-level diagnostic\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "{'classification_scores': {'accuracy': 0.6217162872154116,\n",
            "                           'aopc_thresholds': None,\n",
            "                           'comprehensiveness': 0.2899375320616098,\n",
            "                           'comprehensiveness_aopc': None,\n",
            "                           'comprehensiveness_aopc_points': None,\n",
            "                           'comprehensiveness_entropy': 0.06618782785757517,\n",
            "                           'comprehensiveness_kl': 0.6169633591014108,\n",
            "                           'prf': {'accuracy': 0.6217162872154116,\n",
            "                                   'hatespeech': {'f1-score': 0.776888888888889,\n",
            "                                                  'precision': 0.8229755178907722,\n",
            "                                                  'recall': 0.7356902356902357,\n",
            "                                                  'support': 594},\n",
            "                                   'macro avg': {'f1-score': 0.45722004357298474,\n",
            "                                                 'precision': 0.5202711185762033,\n",
            "                                                 'recall': 0.4112884727239958,\n",
            "                                                 'support': 1142},\n",
            "                                   'normal': {'f1-score': 0.0,\n",
            "                                              'precision': 0.0,\n",
            "                                              'recall': 0.0,\n",
            "                                              'support': 0},\n",
            "                                   'offensive': {'f1-score': 0.5947712418300654,\n",
            "                                                 'precision': 0.7378378378378379,\n",
            "                                                 'recall': 0.4981751824817518,\n",
            "                                                 'support': 548},\n",
            "                                   'weighted avg': {'f1-score': 0.6894979339079474,\n",
            "                                                    'precision': 0.7821213596867371,\n",
            "                                                    'recall': 0.6217162872154116,\n",
            "                                                    'support': 1142}},\n",
            "                           'sufficiency': 0.0014583442993005395,\n",
            "                           'sufficiency_aopc': None,\n",
            "                           'sufficiency_aopc_points': None,\n",
            "                           'sufficiency_entropy': 0.036175898599931006,\n",
            "                           'sufficiency_kl': 0.03766218336473326},\n",
            " 'iou_scores': [{'macro': {'f1': 0.22809206487116668,\n",
            "                           'p': 0.14788382953882084,\n",
            "                           'r': 0.49842381786339757},\n",
            "                 'micro': {'f1': 0.22626646747249762,\n",
            "                           'p': 0.147069209039548,\n",
            "                           'r': 0.49028840494408477},\n",
            "                 'threshold': 0.5}],\n",
            " 'rationale_prf': {'instance_macro': {'f1': 0.11631116013778005,\n",
            "                                      'p': 0.07850262697022813,\n",
            "                                      'r': 0.2640980735551664},\n",
            "                   'instance_micro': {'f1': 0.12033138666304496,\n",
            "                                      'p': 0.0782132768361582,\n",
            "                                      'r': 0.2607416127133608}},\n",
            " 'token_prf': {'instance_macro': {'f1': 0.5074295479278654,\n",
            "                                  'p': 0.6181990659661427,\n",
            "                                  'r': 0.6430023805548021},\n",
            "               'instance_micro': {'f1': 0.44232469993682877,\n",
            "                                  'p': 0.618114406779661,\n",
            "                                  'r': 0.34438323824513084}},\n",
            " 'token_soft_metrics': {'auprc': 0.8384250389403275,\n",
            "                        'average_precision': 0.8342252704971385,\n",
            "                        'roc_auc_score': 0.8541055365711129}}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1eQENR4VLp_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c0a371b-2ecc-40dc-8120-26e64f53520e"
      },
      "source": [
        "# print the required results\r\n",
        "with open('../model_explain_output.json') as fp:\r\n",
        "    output_data = json.load(fp)\r\n",
        "\r\n",
        "print('\\nPlausibility')\r\n",
        "print('IOU F1 :', output_data['iou_scores'][0]['macro']['f1'])\r\n",
        "print('Token F1 :', output_data['token_prf']['instance_macro']['f1'])\r\n",
        "print('AUPRC :', output_data['token_soft_metrics']['auprc'])\r\n",
        "\r\n",
        "print('\\nFaithfulness')\r\n",
        "print('Comprehensiveness :', output_data['classification_scores']['comprehensiveness'])\r\n",
        "print('Sufficiency', output_data['classification_scores']['sufficiency'])"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Plausibility\n",
            "IOU F1 : 0.22809206487116668\n",
            "Token F1 : 0.5074295479278654\n",
            "AUPRC : 0.8384250389403275\n",
            "\n",
            "Faithfulness\n",
            "Comprehensiveness : 0.2899375320616098\n",
            "Sufficiency 0.0014583442993005395\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ND6DYOMxTU8A"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}