Merging_Datasets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multiple-datasets-merge.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "papermill": {
      "duration": 122.337755,
      "end_time": "2021-02-13T17:44:04.381583",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2021-02-13T17:42:02.043828",
      "version": "2.1.0"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWycGV7CYwdT"
      },
      "source": [
        "import os\n",
        "os.environ['KAGGLE_CONFIG_DIR'] = '/content/kaggle'"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HFjanWt_cgCy",
        "outputId": "30a95e0d-38f4-4299-cbcc-4893dbff89a0"
      },
      "source": [
        "!mkdir kaggle/jigsaw kaggle/impermium\n",
        "%cd ./kaggle/jigsaw\n",
        "!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge\n",
        "!unzip train.csv.zip\n",
        "!unzip test.csv.zip\n",
        "!unzip test_labels.csv.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/kaggle/jigsaw\n",
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /content/kaggle/kaggle.json'\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)\n",
            "Downloading test.csv.zip to /content/kaggle/jigsaw\n",
            " 51% 12.0M/23.4M [00:00<00:00, 124MB/s]\n",
            "100% 23.4M/23.4M [00:00<00:00, 115MB/s]\n",
            "Downloading sample_submission.csv.zip to /content/kaggle/jigsaw\n",
            "  0% 0.00/1.39M [00:00<?, ?B/s]\n",
            "100% 1.39M/1.39M [00:00<00:00, 93.7MB/s]\n",
            "Downloading train.csv.zip to /content/kaggle/jigsaw\n",
            " 38% 10.0M/26.3M [00:00<00:00, 102MB/s]\n",
            "100% 26.3M/26.3M [00:00<00:00, 128MB/s]\n",
            "Downloading test_labels.csv.zip to /content/kaggle/jigsaw\n",
            "  0% 0.00/1.46M [00:00<?, ?B/s]\n",
            "100% 1.46M/1.46M [00:00<00:00, 208MB/s]\n",
            "Archive:  train.csv.zip\n",
            "  inflating: train.csv               \n",
            "Archive:  test.csv.zip\n",
            "  inflating: test.csv                \n",
            "Archive:  test_labels.csv.zip\n",
            "  inflating: test_labels.csv         \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8S328Nv4Nqtx",
        "outputId": "f7b2621c-f568-46f7-e624-23e671300dc6"
      },
      "source": [
        "%cd ../impermium\n",
        "!kaggle competitions download -c detecting-insults-in-social-commentary\n",
        "%cd ../.."
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/kaggle/impermium\n",
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /content/kaggle/kaggle.json'\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)\n",
            "Downloading train.csv to /content/kaggle/impermium\n",
            "  0% 0.00/831k [00:00<?, ?B/s]\n",
            "100% 831k/831k [00:00<00:00, 54.8MB/s]\n",
            "Downloading test.csv to /content/kaggle/impermium\n",
            "  0% 0.00/294k [00:00<?, ?B/s]\n",
            "100% 294k/294k [00:00<00:00, 92.2MB/s]\n",
            "Downloading test_with_solutions.csv to /content/kaggle/impermium\n",
            "  0% 0.00/604k [00:00<?, ?B/s]\n",
            "100% 604k/604k [00:00<00:00, 193MB/s]\n",
            "Downloading impermium_verification_set.csv to /content/kaggle/impermium\n",
            "  0% 0.00/323k [00:00<?, ?B/s]\n",
            "100% 323k/323k [00:00<00:00, 105MB/s]\n",
            "Downloading sample_submission_null.csv to /content/kaggle/impermium\n",
            "  0% 0.00/573k [00:00<?, ?B/s]\n",
            "100% 573k/573k [00:00<00:00, 184MB/s]\n",
            "Downloading impermium_verification_labels.csv to /content/kaggle/impermium\n",
            "  0% 0.00/325k [00:00<?, ?B/s]\n",
            "100% 325k/325k [00:00<00:00, 104MB/s]\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJjiLbyVV1H4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "160dc906-1b1a-4e93-cdc8-1750261c1c3a"
      },
      "source": [
        "!wget https://github.com/t-davidson/hate-speech-and-offensive-language/raw/master/data/labeled_data.p\n",
        "\n",
        "!wget https://zenodo.org/record/3950379/files/extended_test-20200717T190516Z-001.zip\n",
        "!unzip extended_test-20200717T190516Z-001.zip\n",
        "\n",
        "!pip install pandarallel"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-04-03 20:39:20--  https://github.com/t-davidson/hate-speech-and-offensive-language/raw/master/data/labeled_data.p\n",
            "Resolving github.com (github.com)... 140.82.112.4\n",
            "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.p [following]\n",
            "--2021-04-03 20:39:21--  https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.p\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.111.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3332233 (3.2M) [application/octet-stream]\n",
            "Saving to: ‘labeled_data.p’\n",
            "\n",
            "labeled_data.p      100%[===================>]   3.18M  20.5MB/s    in 0.2s    \n",
            "\n",
            "2021-04-03 20:39:21 (20.5 MB/s) - ‘labeled_data.p’ saved [3332233/3332233]\n",
            "\n",
            "--2021-04-03 20:39:21--  https://zenodo.org/record/3950379/files/extended_test-20200717T190516Z-001.zip\n",
            "Resolving zenodo.org (zenodo.org)... 137.138.76.77\n",
            "Connecting to zenodo.org (zenodo.org)|137.138.76.77|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 469287 (458K) [application/octet-stream]\n",
            "Saving to: ‘extended_test-20200717T190516Z-001.zip’\n",
            "\n",
            "extended_test-20200 100%[===================>] 458.29K   838KB/s    in 0.5s    \n",
            "\n",
            "2021-04-03 20:39:23 (838 KB/s) - ‘extended_test-20200717T190516Z-001.zip’ saved [469287/469287]\n",
            "\n",
            "Archive:  extended_test-20200717T190516Z-001.zip\n",
            "  inflating: extended_test/test_b_labels_hard.csv  \n",
            "  inflating: extended_test/test_c_labels_hard.csv  \n",
            "  inflating: extended_test/test_c_labels_easy.csv  \n",
            "  inflating: extended_test/test_a_labels_easy.csv  \n",
            "  inflating: extended_test/test_b_labels_all.csv  \n",
            "  inflating: extended_test/test_c_labels_all.csv  \n",
            "  inflating: extended_test/test_b_labels_easy.csv  \n",
            "  inflating: extended_test/test_a_labels_hard.csv  \n",
            "  inflating: extended_test/test_a_labels_all.csv  \n",
            "  inflating: extended_test/test_b_tweets_all.tsv  \n",
            "  inflating: extended_test/test_a_tweets_all.tsv  \n",
            "  inflating: extended_test/test_c_tweets_all.tsv  \n",
            "Requirement already satisfied: pandarallel in /usr/local/lib/python3.7/dist-packages (1.5.2)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from pandarallel) (0.3.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WHAQMx8TV1IZ"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from pandarallel import pandarallel"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-e6iJ1BHV1Ic",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b57f5581-488e-49f2-ba81-aa50b3531548"
      },
      "source": [
        "pandarallel.initialize()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO: Pandarallel will run on 2 workers.\n",
            "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8sM_rZ1V1Ie"
      },
      "source": [
        "label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
        "\n",
        "train_df = pd.read_csv('./kaggle/jigsaw/train.csv')\n",
        "test_df = pd.read_csv('./kaggle/jigsaw/test.csv').merge(pd.read_csv('./kaggle/jigsaw/test_labels.csv'), on='id')\n",
        "test_df.drop(test_df[test_df[label_cols].min(axis=1) == -1].index, inplace=True)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C6THlFBGV1Ig"
      },
      "source": [
        "df = pd.concat([train_df, test_df], ignore_index=True)\n",
        "df['toxic?'] = df.parallel_apply(lambda row: row[label_cols].any(), axis=1)\n",
        "df['dataset'] = 0"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pTLRprTV1Ir"
      },
      "source": [
        "df.drop(label_cols, inplace=True, axis=1)\n",
        "df.drop('id', inplace=True, axis=1)\n",
        "df.rename(columns={'comment_text': 'text'}, inplace=True)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UdFaHTDJV1Iu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4afa236a-4272-4865-b48c-6a51f4de2234"
      },
      "source": [
        "df['toxic?'].mean()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.10050592934882285"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifzoAgQMV1Iw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "1c41ed32-5d6b-49e3-acc1-49643433d324"
      },
      "source": [
        "df"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>toxic?</th>\n",
              "      <th>dataset</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223544</th>\n",
              "      <td>:Jerome, I see you never got around to this…! ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223545</th>\n",
              "      <td>==Lucky bastard== \\n http://wikimediafoundatio...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223546</th>\n",
              "      <td>==shame on you all!!!== \\n\\n You want to speak...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223547</th>\n",
              "      <td>MEL GIBSON IS A NAZI BITCH WHO MAKES SHITTY MO...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223548</th>\n",
              "      <td>\" \\n\\n == Unicorn lair discovery == \\n\\n Suppo...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>223549 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     text  toxic?  dataset\n",
              "0       Explanation\\nWhy the edits made under my usern...       0        0\n",
              "1       D'aww! He matches this background colour I'm s...       0        0\n",
              "2       Hey man, I'm really not trying to edit war. It...       0        0\n",
              "3       \"\\nMore\\nI can't make any real suggestions on ...       0        0\n",
              "4       You, sir, are my hero. Any chance you remember...       0        0\n",
              "...                                                   ...     ...      ...\n",
              "223544  :Jerome, I see you never got around to this…! ...       0        0\n",
              "223545  ==Lucky bastard== \\n http://wikimediafoundatio...       0        0\n",
              "223546  ==shame on you all!!!== \\n\\n You want to speak...       0        0\n",
              "223547  MEL GIBSON IS A NAZI BITCH WHO MAKES SHITTY MO...       1        0\n",
              "223548  \" \\n\\n == Unicorn lair discovery == \\n\\n Suppo...       0        0\n",
              "\n",
              "[223549 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Grv6J86sV1I6"
      },
      "source": [
        "df2 = pd.read_pickle('labeled_data.p')\n",
        "df2.rename(columns={'tweet': 'text'}, inplace=True)\n",
        "df2['toxic?'] = df2.parallel_apply(lambda row: int(row['class'] != 2), axis=1)\n",
        "df2['dataset'] = 1\n",
        "df2 = df2[['text', 'toxic?', 'dataset']]"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ib1RkJbCV1JC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "966406cf-b57d-4c6a-869f-f303259c2d7f"
      },
      "source": [
        "df2['toxic?'].mean()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8320219505306057"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFD-rQqQV1JM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "ed268cde-34fb-4b38-9f7d-6ee9d90225bb"
      },
      "source": [
        "df2"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>toxic?</th>\n",
              "      <th>dataset</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25291</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25292</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25294</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25295</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25296</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>24783 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  toxic?  dataset\n",
              "0      !!! RT @mayasolovely: As a woman you shouldn't...       0        1\n",
              "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...       1        1\n",
              "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...       1        1\n",
              "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...       1        1\n",
              "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...       1        1\n",
              "...                                                  ...     ...      ...\n",
              "25291  you's a muthaf***in lie &#8220;@LifeAsKing: @2...       1        1\n",
              "25292  you've gone and broke the wrong heart baby, an...       0        1\n",
              "25294  young buck wanna eat!!.. dat nigguh like I ain...       1        1\n",
              "25295              youu got wild bitches tellin you lies       1        1\n",
              "25296  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...       0        1\n",
              "\n",
              "[24783 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4n7LUJ-V1JW"
      },
      "source": [
        "df3 = pd.read_csv('onlineHarassmentDataset.tdf', sep='\\t', header=0, encoding='latin-1')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IalRsBjRV1Ja"
      },
      "source": [
        "df3.rename(columns={'Tweet': 'text'}, inplace=True)\n",
        "df3['toxic?'] = df3.parallel_apply(lambda row: int(row['Code'] == 'H'), axis=1)\n",
        "df3['dataset'] = 2\n",
        "df3 = df3[['text', 'toxic?', 'dataset']]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e5PDVVtFV1Jg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a0c57f9-1503-435c-ddc8-f4e20be96998"
      },
      "source": [
        "df3['toxic?'].mean()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.2595776031434185"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KpD_r6QDV1Jh"
      },
      "source": [
        "# df3"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ev6DWTn9V1Jj"
      },
      "source": [
        "train_df4 = pd.read_csv('./kaggle/impermium/train.csv', header=0)\n",
        "test_df4 = pd.read_csv('./kaggle/impermium/test_with_solutions.csv', header=0)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sUrVs9IgV1Jl"
      },
      "source": [
        "df4 = pd.concat([train_df4, test_df4], ignore_index=True)\n",
        "df4.rename(columns={'Comment': 'text', 'Insult': 'toxic?'}, inplace=True)\n",
        "df4['dataset'] = 3\n",
        "df4 = df4[['text', 'toxic?', 'dataset']]"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EyKeGfFPV1Jt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "065abb90-5683-4297-dfde-3962bb05317a"
      },
      "source": [
        "df4['toxic?'].mean()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.26417955717318775"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iSP6B8wuV1Jx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "6345d08c-345f-4abb-f401-ed547ac6dcc7"
      },
      "source": [
        "df4"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>toxic?</th>\n",
              "      <th>dataset</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\"You fuck your dad.\"</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"i really don't understand your point.\\xa0 It ...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\"A\\\\xc2\\\\xa0majority of Canadians can and has ...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\"listen if you dont wanna get married to a man...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"C\\xe1c b\\u1ea1n xu\\u1ed1ng \\u0111\\u01b0\\u1edd...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6589</th>\n",
              "      <td>\"Never really gave it much thought. I just fig...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6590</th>\n",
              "      <td>\"Nadie se salva de la regla 34 xd\"</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6591</th>\n",
              "      <td>\"Question: Are you a boy or a girl?\"</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6592</th>\n",
              "      <td>\"Leave your email or phone number and maybe yo...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6593</th>\n",
              "      <td>\"From the scenarios you present, I see you bel...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6594 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  toxic?  dataset\n",
              "0                                  \"You fuck your dad.\"       1        3\n",
              "1     \"i really don't understand your point.\\xa0 It ...       0        3\n",
              "2     \"A\\\\xc2\\\\xa0majority of Canadians can and has ...       0        3\n",
              "3     \"listen if you dont wanna get married to a man...       0        3\n",
              "4     \"C\\xe1c b\\u1ea1n xu\\u1ed1ng \\u0111\\u01b0\\u1edd...       0        3\n",
              "...                                                 ...     ...      ...\n",
              "6589  \"Never really gave it much thought. I just fig...       0        3\n",
              "6590                 \"Nadie se salva de la regla 34 xd\"       0        3\n",
              "6591               \"Question: Are you a boy or a girl?\"       0        3\n",
              "6592  \"Leave your email or phone number and maybe yo...       1        3\n",
              "6593  \"From the scenarios you present, I see you bel...       0        3\n",
              "\n",
              "[6594 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFzT7DCnV1Jz"
      },
      "source": [
        "df5_tweets = pd.read_csv('./extended_test/test_a_tweets_all.tsv', sep='\\t', header=0)\n",
        "df5_labels = pd.read_csv('./extended_test/test_a_labels_all.csv', names=['id', 'toxic?'])\n",
        "df5 = df5_tweets.merge(df5_labels)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "664pilMUV1J0"
      },
      "source": [
        "df5.rename(columns={'tweet': 'text'}, inplace=True)\n",
        "df5['toxic?'] = df5.parallel_apply(lambda row: int(row['toxic?'] != 'NOT'), axis=1)\n",
        "df5['dataset'] = 4\n",
        "df5 = df5[['text', 'toxic?', 'dataset']]"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JYxPKqLpV1J2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8c0e7767-d682-4853-e5cb-62ab09d1ec5c"
      },
      "source": [
        "df5['toxic?'].mean()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5009177373602536"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GmUaFZyV1J3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "8b74a708-8d99-4959-b6f4-d50195a54471"
      },
      "source": [
        "df5"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>toxic?</th>\n",
              "      <th>dataset</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Has @USER quit? I've not heard of any #knifecr...</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>In celebration of Emancipation Day, we urge yo...</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>@USER @USER It’d be a literal dream come true ...</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Brilliant news to read that Hoggy has signed a...</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>@USER She speaks of the truth 😌</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5988</th>\n",
              "      <td>@USER @USER Nothing about trump is human or no...</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5989</th>\n",
              "      <td>@USER Oh shit. That sounds like a cool time tho</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5990</th>\n",
              "      <td>@USER @USER I gotta say that shit to you one t...</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5991</th>\n",
              "      <td>@USER tRUmp does everything he can to destroy ...</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5992</th>\n",
              "      <td>I was gonna tweet some shit like holding hands...</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5993 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  toxic?  dataset\n",
              "0     Has @USER quit? I've not heard of any #knifecr...       0        4\n",
              "1     In celebration of Emancipation Day, we urge yo...       0        4\n",
              "2     @USER @USER It’d be a literal dream come true ...       0        4\n",
              "3     Brilliant news to read that Hoggy has signed a...       0        4\n",
              "4                       @USER She speaks of the truth 😌       0        4\n",
              "...                                                 ...     ...      ...\n",
              "5988  @USER @USER Nothing about trump is human or no...       1        4\n",
              "5989    @USER Oh shit. That sounds like a cool time tho       1        4\n",
              "5990  @USER @USER I gotta say that shit to you one t...       1        4\n",
              "5991  @USER tRUmp does everything he can to destroy ...       1        4\n",
              "5992  I was gonna tweet some shit like holding hands...       1        4\n",
              "\n",
              "[5993 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qWa4eDRYV1J7"
      },
      "source": [
        "df = pd.concat([df, df2, df3, df4, df5], ignore_index=True)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "503Ruz1xV1J-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4480182-efdc-4eb7-b9d1-a2c7330bdd9f"
      },
      "source": [
        "df.shape, df['toxic?'].mean()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((281279, 3), 0.18884097284191143)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ql3KUPBTV1KA"
      },
      "source": [
        "df.to_pickle('df.pkl')"
      ],
      "execution_count": 29,
      "outputs": []
    }
  ]
}