notebooks/i2i_submission.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "i2i_submission.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Y-UnaSBsYd2",
        "outputId": "0ee82b9c-caba-494e-a7ac-0c74c766a491"
      },
      "source": [
        "!pip install demoji emoji bs4 Levenshtein spacy_langdetect wordsegment xlrd openpyxl"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting demoji\n",
            "  Downloading https://files.pythonhosted.org/packages/88/6a/34379abe01c9c36fe9fddc4181dd935332e7d0159ec3fae76f712e49bcea/demoji-0.4.0-py2.py3-none-any.whl\n",
            "Collecting emoji\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/fa/b3368f41b95a286f8d300e323449ab4e86b85334c2e0b477e94422b8ed0f/emoji-1.2.0-py3-none-any.whl (131kB)\n",
            "\r\u001b[K     |██▌                             | 10kB 28.2MB/s eta 0:00:01\r\u001b[K     |█████                           | 20kB 34.6MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 30kB 39.8MB/s eta 0:00:01\r\u001b[K     |██████████                      | 40kB 39.1MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 51kB 40.3MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 61kB 42.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 71kB 32.5MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 81kB 29.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 92kB 30.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 102kB 28.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 112kB 28.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 122kB 28.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 133kB 28.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: bs4 in /usr/local/lib/python3.7/dist-packages (0.0.1)\n",
            "Collecting Levenshtein\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8e/41/ff25ae28c972a63abde29cd5cea7c648ae0e16b334693cede0522e66dd68/levenshtein-0.12.0-cp37-cp37m-manylinux1_x86_64.whl (158kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 46.5MB/s \n",
            "\u001b[?25hCollecting spacy_langdetect\n",
            "  Downloading https://files.pythonhosted.org/packages/29/70/72dad19abe81ca8e85ff951da170915211d42d705a001d7e353af349a704/spacy_langdetect-0.1.2-py3-none-any.whl\n",
            "Collecting wordsegment\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cf/6c/e6f4734d6f7d28305f52ec81377d7ce7d1856b97b814278e9960183235ad/wordsegment-1.3.1-py2.py3-none-any.whl (4.8MB)\n",
            "\u001b[K     |████████████████████████████████| 4.8MB 47.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: xlrd in /usr/local/lib/python3.7/dist-packages (1.1.0)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.7/dist-packages (2.5.9)\n",
            "Collecting colorama\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: requests<3.0.0 in /usr/local/lib/python3.7/dist-packages (from demoji) (2.23.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from bs4) (4.6.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from Levenshtein) (54.1.2)\n",
            "Collecting langdetect==1.0.7\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/59/4bc44158a767a6d66de18c4136c8aa90491d56cc951c10b74dd1e13213c9/langdetect-1.0.7.zip (998kB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 52.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from spacy_langdetect) (3.6.4)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.7/dist-packages (from openpyxl) (1.0.1)\n",
            "Requirement already satisfied: jdcal in /usr/local/lib/python3.7/dist-packages (from openpyxl) (1.4.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0->demoji) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0->demoji) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0->demoji) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0->demoji) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from langdetect==1.0.7->spacy_langdetect) (1.15.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->spacy_langdetect) (1.10.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->spacy_langdetect) (0.7.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from pytest->spacy_langdetect) (20.3.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->spacy_langdetect) (1.4.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->spacy_langdetect) (8.7.0)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.7-cp37-none-any.whl size=993460 sha256=6bef9d03625c4b6bb4d1af6d129478f619e7217c3b7745ac08e1e2cba1afb089\n",
            "  Stored in directory: /root/.cache/pip/wheels/ec/0c/a9/1647275e7ef5014e7b83ff30105180e332867d65e7617ddafe\n",
            "Successfully built langdetect\n",
            "Installing collected packages: colorama, demoji, emoji, Levenshtein, langdetect, spacy-langdetect, wordsegment\n",
            "Successfully installed Levenshtein-0.12.0 colorama-0.4.4 demoji-0.4.0 emoji-1.2.0 langdetect-1.0.7 spacy-langdetect-0.1.2 wordsegment-1.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FoxEzzP8ZzU5",
        "outputId": "586a7e00-84ba-40cd-ab52-086c0232c4bb"
      },
      "source": [
        "#Basic Python and Machine learning libraries\n",
        "import os, sys, warnings, random, time, re, math, string, demoji, emoji, copy\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import Levenshtein\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "\n",
        "from string import punctuation\n",
        "from collections import Counter\n",
        "from re import search\n",
        "import seaborn as sns\n",
        "from scipy import stats\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn import metrics\n",
        "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS as sklearn_stop_words\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "from nltk.corpus import stopwords \n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import SnowballStemmer, WordNetLemmatizer\n",
        "from nltk.tokenize.casual import casual_tokenize\n",
        "from nltk.util import ngrams\n",
        "\n",
        "import spacy\n",
        "from spacy_langdetect import LanguageDetector\n",
        "\n",
        "warnings.filterwarnings('ignore')\n",
        "demoji.download_codes()\n",
        "\n",
        "#tqdm with pandas\n",
        "from tqdm import tqdm\n",
        "from scipy import stats\n",
        "from tqdm import tqdm_notebook\n",
        "tqdm.pandas()\n",
        "from collections import Counter\n",
        "\n",
        "# sklearn data science models\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.linear_model import LogisticRegression, SGDClassifier, Lasso\n",
        "from sklearn.svm import LinearSVC\n",
        "import xgboost as xgb\n",
        "\n",
        "from wordsegment import load, segment"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading emoji data ...\n",
            "... OK (Got response in 0.14 seconds)\n",
            "Writing emoji data to /root/.demoji/codes.json ...\n",
            "... OK\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0r1J7jlXd-a8"
      },
      "source": [
        "def basic_clean(text):\n",
        "    # text = emoji.demojize(text)\n",
        "    text = str(text)\n",
        "    text = demoji.replace_with_desc(text, sep = \" \") #Emoji to text\n",
        "    text = BeautifulSoup(text, 'lxml').get_text() #links\n",
        "    text = re.sub('https?://\\S+|www\\.\\S+', '', text) #links\n",
        "    text = re.sub(' +', ' ', text) \n",
        "    text = re.sub(\"^RT\", \"\", text) #remove RT\n",
        "    text = re.sub(' +', ' ', text) \n",
        "    text = re.sub(\"^QT\", \"\", text) #remove QT\n",
        "    text = re.sub(' +', ' ', text) \n",
        "    text = re.sub('\\n', \" \", text) #newlines\n",
        "    text = re.sub(' +', ' ', text) \n",
        "    text = str(text).lower() #makes everything lowercase\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                          \"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                          \"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                          \"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                          \"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                          \"\\U0001F700-\\U0001F77F\"  # alchemical symbols\n",
        "                          \"\\U0001F780-\\U0001F7FF\"  # Geometric Shapes Extended\n",
        "                          \"\\U0001F800-\\U0001F8FF\"  # Supplemental Arrows-C\n",
        "                          \"\\U0001F900-\\U0001F9FF\"  # Supplemental Symbols and Pictographs\n",
        "                          \"\\U0001FA00-\\U0001FA6F\"  # Chess Symbols\n",
        "                          \"\\U0001FA70-\\U0001FAFF\"  # Symbols and Pictographs Extended-A\n",
        "                          \"\\U00002702-\\U000027B0\"  # Dingbats\n",
        "                          \"\\U000024C2-\\U0001F251\"\n",
        "                           \"]+\", flags=re.UNICODE)\n",
        "    text = emoji_pattern.sub(r' ', text)\n",
        "    text = re.sub(' +', ' ', text) \n",
        "    text = text.replace('@', ' ').replace('#',' ').replace('_',' ') #removes @ # _ from mentions and user-handles, does not remove the handle itself\n",
        "    return text\n",
        "  \n",
        "def camel_case_split(identifier):\n",
        "  matches = re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', identifier)\n",
        "  return \" \".join([m.group(0) for m in matches])\n",
        "\n",
        "def clean_all(text):\n",
        "    text = basic_clean(text)\n",
        "    text = camel_case_split(text)\n",
        "\n",
        "    return text"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u2-gItJ7ej72",
        "outputId": "a3adcf75-d69c-4d9d-9c77-cb7a1aeb4d2a"
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "stop_words = set(stopwords.words('english'))\n",
        "stemmer = SnowballStemmer('english')\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \n",
        "                       \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \n",
        "                       \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \n",
        "                       \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\",\n",
        "                       \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \n",
        "                       \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\n",
        "                       \"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\n",
        "                       \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \n",
        "                       \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\",\n",
        "                       \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \n",
        "                       \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\n",
        "                       \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\",\n",
        "                       \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\",\n",
        "                       \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\",\n",
        "                       \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\",\n",
        "                       \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \n",
        "                       \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\",\n",
        "                       \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \n",
        "                       \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \n",
        "                       \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n",
        "                       \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\",\n",
        "                       \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'u.s':'america', 'e.g':'for example'}\n",
        "\n",
        "punct = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
        " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
        " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
        " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
        " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n",
        "\n",
        "punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\",\n",
        "                 \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', \n",
        "                 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!':' '}\n",
        "\n",
        "mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater',\n",
        "                'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ',\n",
        "                'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can',\n",
        "                'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', \n",
        "                'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', \n",
        "                'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', \n",
        "                'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization',\n",
        "                'demonetisation': 'demonetization'}\n",
        "\n",
        "def clean_text(text):\n",
        "    '''Clean emoji, Make text lowercase, remove text in square brackets,remove links,remove punctuation\n",
        "    and remove words containing numbers.'''\n",
        "    text = emoji.demojize(text)\n",
        "    text = re.sub(r'\\:(.*?)\\:','',text)\n",
        "    text = str(text).lower()    #Making Text Lowercase\n",
        "    text = re.sub('\\[.*?\\]', '', text)\n",
        "    #The next 2 lines remove html text\n",
        "    text = BeautifulSoup(text, 'lxml').get_text()\n",
        "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
        "    text = re.sub('<.*?>+', '', text)\n",
        "    text = re.sub('\\n', '', text)\n",
        "    text = re.sub('\\w*\\d\\w*', '', text)\n",
        "    # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\", \"'\")\n",
        "    text = re.sub(r\"[^a-zA-Z?.!,¿']+\", \" \", text)\n",
        "    return text\n",
        "\n",
        "def clean_contractions(text, mapping):\n",
        "    '''Clean contraction using contraction mapping'''    \n",
        "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
        "    for s in specials:\n",
        "        text = text.replace(s, \"'\")\n",
        "    for word in mapping.keys():\n",
        "        if \"\"+word+\"\" in text:\n",
        "            text = text.replace(\"\"+word+\"\", \"\"+mapping[word]+\"\")\n",
        "    #Remove Punctuations\n",
        "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
        "    # creating a space between a word and the punctuation following it\n",
        "    # eg: \"he is a boy.\" => \"he is a boy .\"\n",
        "    text = re.sub(r\"([?.!,¿])\", r\" \\1 \", text)\n",
        "    text = re.sub(r'[\" \"]+', \" \", text)\n",
        "    return text\n",
        "\n",
        "def clean_special_chars(text, punct, mapping):\n",
        "    '''Cleans special characters present(if any)'''   \n",
        "    for p in mapping:\n",
        "        text = text.replace(p, mapping[p])\n",
        "    \n",
        "    for p in punct:\n",
        "        text = text.replace(p, f' {p} ')\n",
        "    \n",
        "    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  \n",
        "    for s in specials:\n",
        "        text = text.replace(s, specials[s])\n",
        "    \n",
        "    return text\n",
        "\n",
        "def correct_spelling(x, dic):\n",
        "    '''Corrects common spelling errors'''   \n",
        "    for word in dic.keys():\n",
        "        x = x.replace(word, dic[word])\n",
        "    return x\n",
        "\n",
        "def remove_space(text):\n",
        "    '''Removes awkward spaces'''   \n",
        "    #Removes awkward spaces \n",
        "    text = text.strip()\n",
        "    text = text.split()\n",
        "    return \" \".join(text)\n",
        "\n",
        "def clean_gm(tweet): \n",
        "            \n",
        "    # Special characters\n",
        "    tweet = re.sub(r\"\\x89Û_\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89ÛÒ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89ÛÓ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89ÛÏWhen\", \"When\", tweet)\n",
        "    tweet = re.sub(r\"\\x89ÛÏ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"let\\x89Ûªs\", \"let's\", tweet)\n",
        "    tweet = re.sub(r\"\\x89Û÷\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89Ûª\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89Û\\x9d\", \"\", tweet)\n",
        "    tweet = re.sub(r\"å_\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89Û¢\", \"\", tweet)\n",
        "    tweet = re.sub(r\"\\x89Û¢åÊ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"åÊ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"åÈ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"Ì©\", \"e\", tweet)\n",
        "    tweet = re.sub(r\"å¨\", \"\", tweet)\n",
        "    tweet = re.sub(r\"åÇ\", \"\", tweet)\n",
        "    tweet = re.sub(r\"åÀ\", \"\", tweet)\n",
        "    \n",
        "    return tweet\n",
        "\n",
        "def text_preprocessing_pipeline(text):\n",
        "    '''Cleaning and parsing the text.'''\n",
        "    text = clean_gm(text)\n",
        "    text = clean_text(text)\n",
        "    text = clean_contractions(text, contraction_mapping)\n",
        "    text = clean_special_chars(text, punct, punct_mapping)\n",
        "    text = correct_spelling(text, mispell_dict)\n",
        "    text = remove_space(text)\n",
        "    return text\n",
        "\n",
        "def nlp_preprocessing(text, remove_stopwords=True, stemming=False, lemmatization=True):\n",
        "    '''Stopword removal + Stemming + Lemmatization'''\n",
        "    word_tokens = word_tokenize(text)\n",
        "    if remove_stopwords:\n",
        "        word_tokens = [w for w in word_tokens if not w in stop_words]\n",
        "    if stemming: #Random Cutting, try avoiding it\n",
        "        word_tokens = [stemmer.stem(w) for w in word_tokens]\n",
        "    if lemmatization:\n",
        "        word_tokens = [lemmatizer.lemmatize(w) for w in word_tokens]\n",
        "    return \" \".join(word_tokens)\n",
        "\n",
        "def preprocess_english(pandas_series):\n",
        "    pandas_series = pandas_series.progress_apply(lambda x:text_preprocessing_pipeline(x))\n",
        "    pandas_series = pandas_series.progress_apply(lambda x:nlp_preprocessing(x))\n",
        "\n",
        "    return pandas_series\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DmiUrmByzTfX"
      },
      "source": [
        "# New section"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hoe-IS7be8qn"
      },
      "source": [
        "import torch\n",
        "from torchtext.legacy import data\n",
        "from torch import nn, optim\n",
        "from torch.optim import lr_scheduler"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jirKeT8_fEIq"
      },
      "source": [
        "class config:\n",
        "  TRAIN_ARTICLES = 'Data/labeled_articles.csv'\n",
        "  VAL_ARTICLES = 'Data/labeled_articles.csv'\n",
        "  TEST_ARTICLES = 'Data/labeled_articles.csv'\n",
        "  TRAIN_TWEETS = 'Data/labeled_tweet.csv'\n",
        "  VAL_TWEETS = 'Data/labeled_tweet.csv'\n",
        "  TEST_TWEETS =  'Data/labeled_tweet.csv'\n",
        "  SEED = 713\n",
        "  LOSS = nn.BCEWithLogitsLoss()\n",
        "  LR_SCHEDULER = None\n",
        "  DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "  N_WORKERS = 4\n",
        "  MAX_VOCAB_SIZE = 25_000\n",
        "  BATCH_SIZE = 64\n",
        "  EMBEDDINGS = \"glove.6B.100d\"\n",
        "  EMBEDDING_DIM = 100\n",
        "  HIDDEN_DIM = 256\n",
        "  OUTPUT_DIM = 1\n",
        "  N_LAYERS = 2\n",
        "  BIDIRECTIONAL = True\n",
        "  DROPOUT = 0.5\n",
        "  N_EPOCHS = 70   "
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xVP294upfJdE"
      },
      "source": [
        "from torchtext.legacy import data\n",
        "\n",
        "class DataFrameDataset(data.Dataset):\n",
        "\n",
        "    def __init__(self, df, text_field, label_field, is_test=False, **kwargs):\n",
        "        fields = [('Text', text_field), ('label', label_field)]\n",
        "        examples = []\n",
        "        for i, row in df.iterrows():\n",
        "            label = row.label if not is_test else None\n",
        "            text = row.Text\n",
        "            examples.append(data.Example.fromlist([text, label], fields))\n",
        "\n",
        "        super().__init__(examples, fields, **kwargs)\n",
        "\n",
        "    @staticmethod\n",
        "    def sort_key(ex):\n",
        "        return len(ex.Text)\n",
        "\n",
        "    @classmethod\n",
        "    def splits(cls, text_field, label_field, train_df, val_df=None, test_df=None, **kwargs):\n",
        "        train_data, val_data, test_data = (None, None, None)\n",
        "\n",
        "        if train_df is not None:\n",
        "            train_data = cls(train_df.copy(), text_field, label_field, **kwargs)\n",
        "        if val_df is not None:\n",
        "            val_data = cls(val_df.copy(), text_field, label_field, **kwargs)\n",
        "        if test_df is not None:\n",
        "            test_data = cls(test_df.copy(), text_field, label_field, True, **kwargs)\n",
        "\n",
        "        return tuple(d for d in (train_data, val_data, test_data) if d is not None)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V_cy5EV2fsSH"
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "\n",
        "class RNN_TWEET(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
        "                 bidirectional, dropout, pad_idx):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)\n",
        "        \n",
        "        self.rnn = nn.LSTM(embedding_dim, \n",
        "                           hidden_dim, \n",
        "                           num_layers=n_layers, \n",
        "                           bidirectional=bidirectional, \n",
        "                           dropout=dropout)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, text, text_lengths):\n",
        "        \n",
        "        #text = [sent len, batch size]\n",
        "        \n",
        "        embedded = self.dropout(self.embedding(text))\n",
        "        \n",
        "        #embedded = [sent len, batch size, emb dim]\n",
        "        \n",
        "        #pack sequence\n",
        "        # lengths need to be on CPU!\n",
        "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths.to('cpu'))\n",
        "        \n",
        "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
        "        \n",
        "        #unpack sequence\n",
        "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
        "\n",
        "        #output = [sent len, batch size, hid dim * num directions]\n",
        "        #output over padding tokens are zero tensors\n",
        "        \n",
        "        #hidden = [num layers * num directions, batch size, hid dim]\n",
        "        #cell = [num layers * num directions, batch size, hid dim]\n",
        "        \n",
        "        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers\n",
        "        #and apply dropout\n",
        "        \n",
        "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
        "                \n",
        "        #hidden = [batch size, hid dim * num directions]\n",
        "            \n",
        "        return self.fc(hidden)\n"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJpveRLAfUE1"
      },
      "source": [
        "import warnings, random, time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "from sklearn import metrics\n",
        "\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs\n",
        "\n",
        "def binary_accuracy(preds, y):\n",
        "    \"\"\"\n",
        "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
        "    \"\"\"\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc\n",
        "\n",
        "def data_lao():        \n",
        "    train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "        (train_ds, val_ds, test_ds), \n",
        "        batch_size = BATCH_SIZE,\n",
        "        sort_within_batch = True,\n",
        "        device = device)\n",
        "\n",
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        text, text_lengths = batch.text\n",
        "        \n",
        "        predictions = model(text, text_lengths).squeeze(1)\n",
        "        \n",
        "        loss = criterion(predictions, batch.label)\n",
        "        \n",
        "        acc = binary_accuracy(predictions, batch.label)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text, text_lengths = batch.text\n",
        "            \n",
        "            predictions = model(text, text_lengths).squeeze(1)\n",
        "            \n",
        "            loss = criterion(predictions, batch.label)\n",
        "            \n",
        "            acc = binary_accuracy(predictions, batch.label)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "# if __name__ == '__main__':\n",
        "# #     tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)\n",
        "# #     model = BertForSequenceClassification.from_pretrained(\n",
        "# #     'bert-large-uncased', # Use the 124-layer, 1024-hidden, 16-heads, 340M parameters BERT model with an uncased vocab.\n",
        "# #     num_labels = 2, # The number of output labels--2 for binary classification. You can increase this for multi-class tasks.   \n",
        "# #     output_attentions = False, # Whether the model returns attentions weights.\n",
        "# #     output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        "# # )\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vAkleXlvfnwW"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "# import ipdb\n",
        "import spacy \n",
        "import torch, time\n",
        "from tqdm import tqdm\n",
        "from torch import nn, optim\n",
        "from torchtext.legacy import data\n",
        "\n",
        "\n",
        "tqdm.pandas()\n",
        "\n",
        "def run():\n",
        "    training_df = pd.read_csv('Data/labeled_tweet.csv')\n",
        "    # ipdb.set_trace()\n",
        "    training_df['text'] = training_df['text'].apply(clean_all)\n",
        "    validation_df = pd.read_csv('Data/labeled_tweet.csv')\n",
        "    validation_df['text'] = validation_df['text'].apply(clean_all)\n",
        "\n",
        "    TEXT = data.Field(tokenize = 'spacy',\n",
        "                    tokenizer_language = 'en_core_web_sm',\n",
        "                    include_lengths = True)\n",
        "    LABEL = data.LabelField(dtype = torch.float)\n",
        "\n",
        "    train_ds, val_ds, test_ds = DataFrameDataset.splits(\n",
        "        text_field=TEXT, label_field=LABEL, train_df=training_df, val_df=validation_df, test_df=validation_df)\n",
        "    # ipdb.set_trace()    \n",
        "    TEXT.build_vocab(train_ds, max_size = config.MAX_VOCAB_SIZE, vectors = config.EMBEDDINGS, \n",
        "                 unk_init = torch.Tensor.normal_)\n",
        "    LABEL.build_vocab(train_ds)\n",
        "    \n",
        "    train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "        (train_ds, val_ds, test_ds), \n",
        "        batch_size = config.BATCH_SIZE,\n",
        "        sort_within_batch = True,\n",
        "        device = config.DEVICE)\n",
        "\n",
        "    pretrained_embeddings = TEXT.vocab.vectors\n",
        "    INPUT_DIM = len(TEXT.vocab)\n",
        "    PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
        "    UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "\n",
        "    model = RNN_TWEET(INPUT_DIM, \n",
        "                config.EMBEDDING_DIM, \n",
        "                config.HIDDEN_DIM, \n",
        "                config.OUTPUT_DIM, \n",
        "                config.N_LAYERS, \n",
        "                config.BIDIRECTIONAL, \n",
        "                config.DROPOUT, \n",
        "                PAD_IDX)\n",
        "    optimizer = optim.Adam(model.parameters())\n",
        "    criterion = config.LOSS\n",
        "    criterion = criterion.to(config.DEVICE)\n",
        "\n",
        "    model.embedding.weight.data.copy_(pretrained_embeddings)\n",
        "    model.embedding.weight.data[UNK_IDX] = torch.zeros(config.EMBEDDING_DIM)\n",
        "    model.embedding.weight.data[PAD_IDX] = torch.zeros(config.EMBEDDING_DIM)\n",
        "\n",
        "    model = model.to(config.DEVICE)\n",
        "\n",
        "    best_valid_loss = float('inf')\n",
        "\n",
        "    for epoch in range(config.N_EPOCHS):\n",
        "\n",
        "        start_time = time.time()\n",
        "        \n",
        "        train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "        valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "        \n",
        "        end_time = time.time()\n",
        "\n",
        "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "        \n",
        "        if valid_loss < best_valid_loss:\n",
        "            best_valid_loss = valid_loss\n",
        "            torch.save(model.state_dict(), 'Models/rnn_tweet.pt')\n",
        "        \n",
        "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "        print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "mPvAFTq6nYD8",
        "outputId": "bb7fbce9-9084-4494-9489-c6ee2b43dd05"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/Submission/Train_Data/evaluation_data_corrected.csv')\n",
        "df.head()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text_ID</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>article_4001</td>\n",
              "      <td>Market Highlights\\n\\nAn automotive constant ve...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>article_4002</td>\n",
              "      <td>Overview:\\n\\nBe it as a means for food or as p...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>article_4003</td>\n",
              "      <td>GenScript USA Inc. (U.S.), Horizon Discovery G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>article_4004</td>\n",
              "      <td>New Delhi: Prime Minister Narendra Modi spoke ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>article_4005</td>\n",
              "      <td>New Delhi, Feb 1 (PTI) The Delhi High Court Mo...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Text_ID                                               Text\n",
              "0  article_4001  Market Highlights\\n\\nAn automotive constant ve...\n",
              "1  article_4002  Overview:\\n\\nBe it as a means for food or as p...\n",
              "2  article_4003  GenScript USA Inc. (U.S.), Horizon Discovery G...\n",
              "3  article_4004  New Delhi: Prime Minister Narendra Modi spoke ...\n",
              "4  article_4005  New Delhi, Feb 1 (PTI) The Delhi High Court Mo..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dSgVB_R4nlbe"
      },
      "source": [
        "df.Text = df.Text.apply(clean_all)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhX1AKnmoXZN"
      },
      "source": [
        "df_article, df_tweet = df.iloc[:222, :], df.iloc[222:, :]"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZQBSq9gIf6Ls"
      },
      "source": [
        "def predict_sentiment(model, sentence):\n",
        "    nlp = spacy.load('en_core_web_sm')\n",
        "    model.eval()\n",
        "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]\n",
        "    indexed = [TEXT.vocab.stoi[t] for t in tokenized]\n",
        "    length = [len(indexed)]\n",
        "    device = 'cuda'\n",
        "    tensor = torch.LongTensor(indexed).to(device)\n",
        "    tensor = tensor.unsqueeze(1)\n",
        "    length_tensor = torch.LongTensor(length)\n",
        "    prediction = torch.sigmoid(model(tensor, length_tensor))\n",
        "    return np.round(prediction.item())"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZPqfPPO-iSkX"
      },
      "source": [
        "import pickle\n",
        "def save_vocab(vocab, path):\n",
        "    output = open(path, 'wb')\n",
        "    pickle.dump(vocab, output)\n",
        "    output.close()\n",
        "def load_vocab(path):\n",
        "    output = open(path, 'rb')\n",
        "    vocab = pickle.load(output)\n",
        "    return vocab"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "itt6Dvi2f79K",
        "outputId": "b3c7b236-46af-49b2-8761-3a8ad003b4f2"
      },
      "source": [
        "TEXT = data.Field(tokenize = 'spacy',\n",
        "                tokenizer_language = 'en_core_web_sm',\n",
        "                include_lengths = True)\n",
        "LABEL = data.LabelField(dtype = torch.float)\n",
        "TEXT.vocab = load_vocab('/content/drive/MyDrive/Submission/Classification/Models/article_text_vocab.pkl')\n",
        "LABEL.vocab = load_vocab('/content/drive/MyDrive/Submission/Classification/Models/article_label_vocab.pkl')\n",
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "INPUT_DIM = len(TEXT.vocab)\n",
        "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
        "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "model_article = RNN_TWEET(INPUT_DIM, \n",
        "                config.EMBEDDING_DIM, \n",
        "                config.HIDDEN_DIM, \n",
        "                config.OUTPUT_DIM, \n",
        "                config.N_LAYERS, \n",
        "                config.BIDIRECTIONAL, \n",
        "                config.DROPOUT, \n",
        "                PAD_IDX)\n",
        "model_article.load_state_dict(torch.load('/content/drive/MyDrive/Submission/Classification/Models/rnn_article.pt'))\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mMLrKSOPtJVk",
        "outputId": "8fdeccaf-fb32-4a11-da4f-bc60fc209e22"
      },
      "source": [
        "device = 'cuda'\n",
        "model_article.to(device)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RNN_TWEET(\n",
              "  (embedding): Embedding(17685, 100, padding_idx=1)\n",
              "  (rnn): LSTM(100, 256, num_layers=2, dropout=0.5, bidirectional=True)\n",
              "  (fc): Linear(in_features=512, out_features=1, bias=True)\n",
              "  (dropout): Dropout(p=0.5, inplace=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0T1pgJvG4F0A",
        "outputId": "64b821f7-aaf1-422e-f153-c373a7979572"
      },
      "source": [
        "TEXT = data.Field(tokenize = 'spacy',\n",
        "                tokenizer_language = 'en_core_web_sm',\n",
        "                include_lengths = True)\n",
        "LABEL = data.LabelField(dtype = torch.float)\n",
        "TEXT.vocab = load_vocab('/content/drive/MyDrive/Submission/Classification/Models/tweet_text_vocab.pkl')\n",
        "LABEL.vocab = load_vocab('/content/drive/MyDrive/Submission/Classification/Models/tweet_label_vocab.pkl')\n",
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "INPUT_DIM = len(TEXT.vocab)\n",
        "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
        "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "model_tweet = RNN_TWEET(INPUT_DIM, \n",
        "                config.EMBEDDING_DIM, \n",
        "                config.HIDDEN_DIM, \n",
        "                config.OUTPUT_DIM, \n",
        "                config.N_LAYERS, \n",
        "                config.BIDIRECTIONAL, \n",
        "                config.DROPOUT, \n",
        "                PAD_IDX)\n",
        "model_tweet.load_state_dict(torch.load('/content/drive/MyDrive/Submission/Classification/Models/rnn_tweet.pt'))\n",
        "model_tweet.to(device)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RNN_TWEET(\n",
              "  (embedding): Embedding(3045, 100, padding_idx=1)\n",
              "  (rnn): LSTM(100, 256, num_layers=2, dropout=0.5, bidirectional=True)\n",
              "  (fc): Linear(in_features=512, out_features=1, bias=True)\n",
              "  (dropout): Dropout(p=0.5, inplace=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OK8f4marh0wl"
      },
      "source": [
        "def append_sentiment_labels(df, model):\n",
        "  df['Mobile_Tech'] = np.zeros(len(df))\n",
        "  for i in range(len(df)):\n",
        "    df.Mobile_Tech.iloc[i] = predict_sentiment(model, df.Text.iloc[i])\n"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5FZVOpk5sgpZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4cdee5bc-c18e-4979-e6bb-40b7b5b3228f"
      },
      "source": [
        "%%time\n",
        "append_sentiment_labels(df_article, model_article)\n",
        "append_sentiment_labels(df_tweet, model_tweet)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3min 37s, sys: 3.14 s, total: 3min 40s\n",
            "Wall time: 3min 40s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "HVN9D6xLtA65",
        "outputId": "befaec36-758c-4729-e163-2eb5284896fa"
      },
      "source": [
        "df_article"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text_ID</th>\n",
              "      <th>Text</th>\n",
              "      <th>Mobile_Tech</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>article_4001</td>\n",
              "      <td>market highlights an automotive constant veloc...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>article_4002</td>\n",
              "      <td>overview: be it as a means for food or as pets...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>article_4003</td>\n",
              "      <td>genscript usa inc. (u.s.), horizon discovery g...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>article_4004</td>\n",
              "      <td>new delhi: prime minister narendra modi spoke ...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>article_4005</td>\n",
              "      <td>new delhi, feb 1 (pti) the delhi high court mo...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>article_4220</td>\n",
              "      <td>motorola released an update to fix one of the ...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>article_4221</td>\n",
              "      <td>the company has rolled out software update to ...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>article_4222</td>\n",
              "      <td>some xiaomi and redmi devices are facing a pec...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220</th>\n",
              "      <td>article_4223</td>\n",
              "      <td>mi, redmi, and poco users are facing an issue ...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>221</th>\n",
              "      <td>article_4224</td>\n",
              "      <td>if you inquire for a good phone, almost all ti...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>222 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          Text_ID  ... Mobile_Tech\n",
              "0    article_4001  ...         0.0\n",
              "1    article_4002  ...         0.0\n",
              "2    article_4003  ...         0.0\n",
              "3    article_4004  ...         1.0\n",
              "4    article_4005  ...         1.0\n",
              "..            ...  ...         ...\n",
              "217  article_4220  ...         0.0\n",
              "218  article_4221  ...         0.0\n",
              "219  article_4222  ...         0.0\n",
              "220  article_4223  ...         0.0\n",
              "221  article_4224  ...         0.0\n",
              "\n",
              "[222 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6DzrNEiuz7A",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "0b6a2608-ac9b-4147-be47-7ff48f4cfc2f"
      },
      "source": [
        "df_tweet"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text_ID</th>\n",
              "      <th>Text</th>\n",
              "      <th>Mobile_Tech</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>222</th>\n",
              "      <td>tweet_4001</td>\n",
              "      <td>lloydsbank how do i report a scam on my accou...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223</th>\n",
              "      <td>tweet_4002</td>\n",
              "      <td>iameraijaz: i have been denied security in j...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>tweet_4003</td>\n",
              "      <td>fplprop  gooner fpl i think there's a risk sa...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>225</th>\n",
              "      <td>tweet_4004</td>\n",
              "      <td>brfootball: official: bayern munich loan def...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>226</th>\n",
              "      <td>tweet_4005</td>\n",
              "      <td>timinhonolulu: 5. so far, three 2016 cycle u...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>427</th>\n",
              "      <td>tweet_4206</td>\n",
              "      <td>ios 6.1  for  iphone  very  bad and  iphone5 ...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>428</th>\n",
              "      <td>tweet_4207</td>\n",
              "      <td>one  bad thing of  xiaomiredminote4 is the aut...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>429</th>\n",
              "      <td>tweet_4208</td>\n",
              "      <td>i have a concern over the service provided . p...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>430</th>\n",
              "      <td>tweet_4209</td>\n",
              "      <td>bought nokia6.1plus in oct2018 and having wor...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>431</th>\n",
              "      <td>tweet_4210</td>\n",
              "      <td>now switching to  oneplus in  oneplus6 .. real...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>210 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        Text_ID                                               Text  Mobile_Tech\n",
              "222  tweet_4001   lloydsbank how do i report a scam on my accou...          0.0\n",
              "223  tweet_4002    iameraijaz: i have been denied security in j...          0.0\n",
              "224  tweet_4003   fplprop  gooner fpl i think there's a risk sa...          0.0\n",
              "225  tweet_4004    brfootball: official: bayern munich loan def...          0.0\n",
              "226  tweet_4005    timinhonolulu: 5. so far, three 2016 cycle u...          0.0\n",
              "..          ...                                                ...          ...\n",
              "427  tweet_4206   ios 6.1  for  iphone  very  bad and  iphone5 ...          1.0\n",
              "428  tweet_4207  one  bad thing of  xiaomiredminote4 is the aut...          1.0\n",
              "429  tweet_4208  i have a concern over the service provided . p...          1.0\n",
              "430  tweet_4209   bought nokia6.1plus in oct2018 and having wor...          1.0\n",
              "431  tweet_4210  now switching to  oneplus in  oneplus6 .. real...          1.0\n",
              "\n",
              "[210 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qEEwqUexFW3Q",
        "outputId": "d1eb1ad1-c126-4f7c-8b22-64838cf30714"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zxLw29MG8LJf"
      },
      "source": [
        "company = ['10.or',\n",
        " 'acer',\n",
        " 'adcom',\n",
        " 'airtel',\n",
        " 'alcatel',\n",
        " 'alpha',\n",
        " 'amazon',\n",
        " 'aoc',\n",
        " 'apple',\n",
        " 'aqua',\n",
        " 'archos',\n",
        " 'asus',\n",
        " 'benq',\n",
        " 'billion',\n",
        " 'black shark',\n",
        " 'blackberry',\n",
        " 'blu',\n",
        " 'bq',\n",
        " 'byond',\n",
        " 'celkon',\n",
        " 'centric',\n",
        " 'champion computers',\n",
        " 'champone',\n",
        " 'cherry mobile',\n",
        " 'comio',\n",
        " 'coolpad',\n",
        " 'creo',\n",
        " 'croma',\n",
        " 'datawind',\n",
        " 'detel',\n",
        " 'docoss',\n",
        " 'doogee',\n",
        " 'elari',\n",
        " 'elephone',\n",
        " 'energizer',\n",
        " 'essential',\n",
        " 'evercoss',\n",
        " 'fairphone',\n",
        " 'geeksphone',\n",
        " 'general mobile',\n",
        " 'gionee',\n",
        " 'google',\n",
        " 'homtom',\n",
        " 'honor',\n",
        " 'hp',\n",
        " 'htc',\n",
        " 'huawei',\n",
        " 'huawai',\n",
        " 'hyve',\n",
        " 'i-mobiles',\n",
        " 'iball',\n",
        " 'iberry',\n",
        " 'infinix',\n",
        " 'infocus',\n",
        " 'intex',\n",
        " 'itel',\n",
        " 'ivoomi',\n",
        " 'jio',\n",
        " 'jivi',\n",
        " 'jolla',\n",
        " 'josh mobile',\n",
        " 'karbonn',\n",
        " 'kestrel',\n",
        " 'kodak',\n",
        " 'kult',\n",
        " 'kyocera',\n",
        " 'land rover',\n",
        " 'lava',\n",
        " 'lemon',\n",
        " 'lenovo',\n",
        " 'lg',\n",
        " 'lumigon',\n",
        " 'm-tech',\n",
        " 'mafe mobile',\n",
        " 'magicon',\n",
        " 'marshall',\n",
        " 'maxx mobile',\n",
        " 'meizu',\n",
        " 'mercury',\n",
        " 'micromax',\n",
        " 'microsoft',\n",
        " 'mitashi',\n",
        " 'mito',\n",
        " 'mobiistar',\n",
        " 'motorola',\n",
        " 'mphone',\n",
        " 'mts',\n",
        " 'myphone',\n",
        " 'namotel',\n",
        " 'neffos',\n",
        " 'nexian',\n",
        " 'nextbit',\n",
        " 'nokia',\n",
        " 'nubia',\n",
        " 'nuu mobile',\n",
        " 'obi worldphone',\n",
        " 'oneplus',\n",
        " 'onida',\n",
        " 'oplus',\n",
        " 'oppo',\n",
        " 'oukitel',\n",
        " 'panasonic',\n",
        " 'pantel',\n",
        " 'pepsi',\n",
        " 'phicomm',\n",
        " 'philips',\n",
        " 'poco',\n",
        " 'polaroid',\n",
        " 'porsche design',\n",
        " 'qiku',\n",
        " 'qmobile',\n",
        " 'razer',\n",
        " 'reach',\n",
        " 'realme',\n",
        " 'reliance',\n",
        " 'ringing bells',\n",
        " 'rio',\n",
        " 'royole',\n",
        " 'salora',\n",
        " 'samsung',\n",
        " 'sansui',\n",
        " 'saygus',\n",
        " 'sharp',\n",
        " 'sikur',\n",
        " 'silent circle',\n",
        " 'simmtronics',\n",
        " 'sirin',\n",
        " 'smartisan',\n",
        " 'smartron',\n",
        " 'sony',\n",
        " 'spice',\n",
        " 'sunstrike',\n",
        " 'swipe',\n",
        " 'symphony',\n",
        " 't mobile',\n",
        " 'tambo',\n",
        " 'tcl',\n",
        " 'tecno',\n",
        " 'tonino lamborghini',\n",
        " 'unihertz',\n",
        " 'vaio',\n",
        " 'vertu',\n",
        " 'viaan',\n",
        " 'videocon',\n",
        " 'vivo',\n",
        " 'vodafone',\n",
        " 'wickedleak',\n",
        " 'wiio',\n",
        " 'xiaomi',\n",
        " 'xolo',\n",
        " 'yamada denki',\n",
        " 'yandex',\n",
        " 'yota devices',\n",
        " 'yu',\n",
        " 'zen',\n",
        " 'ziox',\n",
        " 'zopo',\n",
        " 'zte',\n",
        " 'zuk',\n",
        " 'zync']"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsP44cpR9uOB"
      },
      "source": [
        "map_sentiments = {2:'positive',1:'neutral',0:'negative'}"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "9A-SIAtSJROK",
        "outputId": "10648d8c-fb57-4c43-e537-19967bb50977"
      },
      "source": [
        "pwd"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/MyDrive/Submission'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yadJzkmvJKnR",
        "outputId": "85285ae2-10ea-45eb-f647-269097fe9b21"
      },
      "source": [
        "%cd /content/drive/MyDrive/Submission"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/Submission\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cR5RBwcA9jZ0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "outputId": "6554fce6-c93e-44a4-8242-574c02cf5592"
      },
      "source": [
        "from ASGCN.infer import Inferer\n",
        "\n",
        "\n",
        "Infer = Inferer()\n",
        "allOutput = pd.DataFrame()"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-45-d3bf19d7d3ab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mASGCN\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInferer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mInfer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInferer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mallOutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'ASGCN'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RxbcPgc1LdD1"
      },
      "source": [
        "pat = re.compile(\"|\".join(map(re.escape, company))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYjAiYpp8fc_"
      },
      "source": [
        "for i in range(len(df_tweet)):\n",
        "  text = df_tweet.Text.iloc[i]\n",
        "  predicted_brands = list(set(pat.findall(text)))\n",
        "  temp_df = pd.DataFrame()\n",
        "  mobile_tech = df_tweet['Mobile_Tech'][i]\n",
        "  if mobile_tech==1:\n",
        "    for j in range(len(predicted_brands)):\n",
        "      brand = predicted_brands[j]\n",
        "      ABSA_model_out = Infer.evaluate(text,brand)\n",
        "      sentiment = map_sentiments[ABSA_model_out]\n",
        "      temp_df1 = pd.DataFrame({'Text_ID': df_tweet['Text_ID'][i],\n",
        "                            'Mobile_Tech': mobile_tech,\n",
        "                            'Brands_Entity_Identified':brand, \n",
        "                            'Sentiment_Identified':sentiment,}, index = [0])    \n",
        "      temp_df = temp_df.append(temp_df1)\n",
        "  else:\n",
        "    brand = \"\"\n",
        "    sentiment = \"\"\n",
        "    temp_df1 = pd.DataFrame({'Text_ID': df_tweet['Text_ID'][i],\n",
        "                            'Mobile_Tech': mobile_tech,\n",
        "                            'Brands_Entity_Identified':brand, \n",
        "                            'Sentiment_Identified':sentiment,}, index = [0])    \n",
        "    temp_df = temp_df.append(temp_df1)\n",
        "  allOutput = allOutput.append(temp_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "koOCIcmg_ABW"
      },
      "source": [
        "for i in range(len(df_article)):\n",
        "  text = df_article.Text.iloc[i]\n",
        "  predicted_brands = list(set(pat.findall(text)))\n",
        "  temp_df = pd.DataFrame()\n",
        "  if mobile_tech==1:\n",
        "    for j in range(len(predicted_brands)):\n",
        "      brand = predicted_brands[j]\n",
        "      ABSA_model_out = Infer.evaluate(text,brand)\n",
        "      sentiment = map_sentiments[ABSA_model_out]\n",
        "      temp_df1 = pd.DataFrame({'Text_ID': df_article['Text_ID'][i],\n",
        "                            'Mobile_Tech': mobile_tech,\n",
        "                            'Brands_Entity_Identified':brand, \n",
        "                            'Sentiment_Identified':sentiment,}, index = [0])    \n",
        "      temp_df = temp_df.append(temp_df1)\n",
        "  else:\n",
        "    brand = \"\"\n",
        "    sentiment = \"\"\n",
        "    temp_df1 = pd.DataFrame({'Text_ID': df_article['Text_ID'][i],\n",
        "                            'Mobile_Tech': mobile_tech,\n",
        "                            'Brands_Entity_Identified':brand, \n",
        "                            'Sentiment_Identified':sentiment,}, index = [0]) \n",
        "  allOutput = allOutput.append(temp_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hN2oNW-uDmXo"
      },
      "source": [
        "allOutput.to_csv(\"output2.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Q5u40vF_wL1"
      },
      "source": [
        "from heading_generation import generate_headlines\n",
        "import copy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qxUvPv6wAjg6"
      },
      "source": [
        "labels = df_article[\"Mobile_Tech\"]==1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dC06JEoC__8z"
      },
      "source": [
        "df_heading  = copy.copy(df_article[labels])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_QDr994r_Z4c"
      },
      "source": [
        "headlines = generate_headlines(dataframe = df_headling, PATH = './models/news.pth')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cP6My4neAez_"
      },
      "source": [
        "df_heading[\"Heading\"] = headlines"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t7mkBUuFB7mE"
      },
      "source": [
        "df_article[\"Heading\"] = [\"\"]*len(df_article)\n",
        "df_tweet[\"Heading\"] = [\"\"]*len(df_tweet)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x8EiplDiCu7u"
      },
      "source": [
        "df_article = df_article.drop(~labels, axis=0)\n",
        "df_article.append(df_heading)\n",
        "df_final = pd.DataFrame()\n",
        "df_final.append(df_tweet)\n",
        "df_final.append(df_article)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IaJ_YdJzDZQd"
      },
      "source": [
        "df_final.to_csv(\"output1.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}