input_marge_v3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "input_marge_v3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMoIxMUPCRFZSQFyVU4RQxC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ronakdm/input-marginalization/blob/main/input_marge_v3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AxXBIE5Cw7dT"
      },
      "source": [
        "%%capture\n",
        "!pip install pytorch_pretrained_bert\n",
        "!pip install transformers\n",
        "!git clone https://github.com/ronakdm/input-marginalization.git"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GczF6JdTxhZo",
        "outputId": "db4c8aed-de64-4474-ae8f-a780090ed2cd"
      },
      "source": [
        "%%bash\n",
        "cd input-marginalization\n",
        "git pull\n",
        "cd .."
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Already up to date.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2NYs_qKDxnST"
      },
      "source": [
        "import sys\n",
        "sys.path.append(\"input-marginalization\")\n",
        "import torch\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
        "from transformers import BertTokenizer, BertModel\n",
        "from utils import generate_dataloaders\n",
        "from models import LSTM\n",
        "from torch.nn import LogSoftmax\n",
        "import math\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i2T8Fwh5xqqr",
        "outputId": "755c2356-098d-4926-e401-8e1243e69cfd"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive',force_remount=True)\n",
        "save_dir = \"/content/gdrive/My Drive/input-marginalization\""
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3XZW2qJx1qy"
      },
      "source": [
        "SAMPLE_SIZE = 5\n",
        "SIGMA = 1e-4\n",
        "log_softmax = LogSoftmax(dim=0)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAwYhg_Dy5SC"
      },
      "source": [
        "%%capture\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "bert_model = BertForMaskedLM.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w8bCqegYx4yk"
      },
      "source": [
        "bert = torch.load(f\"{save_dir}/bert_sst2.pt\")\n",
        "cnn = torch.load(f\"{save_dir}/cnn_sst2.pt\")\n",
        "lstm = torch.load(f\"{save_dir}/lstm_sst2.pt\")"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dtx2h5-7y0Sk"
      },
      "source": [
        "def loaddata ():\n",
        "  train_dataloader, validation_dataloader, test_dataloader = generate_dataloaders(1)\n",
        "  return test_dataloader"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eM8YEA2aylyP"
      },
      "source": [
        "def compute_probability(model, input_ids, attention_masks, label):\n",
        "    logits = model(\n",
        "        input_ids, token_type_ids=None, attention_mask=attention_masks, labels=label.repeat((len(input_ids))),\n",
        "    ).logits\n",
        "    \n",
        "    return math.exp(logits[0][label])\n",
        "    "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g3vC5A-8iJey"
      },
      "source": [
        "def compute_probability2(model, input_ids, attention_masks, label):\n",
        "  \n",
        "    logits = model(\n",
        "        input_ids.to(torch.int64), token_type_ids=None, attention_mask=attention_masks, labels=label.repeat((len(input_ids))),\n",
        "    ).logits\n",
        "\n",
        "    return torch.exp(torch.reshape(logits[:, label], (-1,)))\n",
        "    "
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "itGpt67FymBv"
      },
      "source": [
        "def calculate_woe(model, input_ids, attention_masks, label, sigma):\n",
        "  device = \"cuda\" if next(model.parameters()).is_cuda else \"cpu\"\n",
        "  bert_model.to(device)\n",
        "  \n",
        "  #predictions is the probability distribution of each word in the vocabulary for each word in input sentence\n",
        "  predictions = bert_model(input_ids)\n",
        "  predictions = torch.squeeze(predictions)\n",
        "  predictions = F.softmax(predictions, dim=1)\n",
        "\n",
        "  #woe is the weight of evidence\n",
        "  woe = []\n",
        "  model.eval()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for j in range (len(predictions)):\n",
        "      word_scores = predictions[j]\n",
        "      input_batch = input_ids.clone().to(device)\n",
        "      \n",
        "      #word_scores_batch calculates the value of the MLM of Bert for each masked word\n",
        "      #we put 0 for the first input which is unmasked\n",
        "      word_scores_batch = [0]\n",
        "\n",
        "      for k in range(len(word_scores)):\n",
        "        if word_scores[k] > sigma:\n",
        "           input_batch = torch.cat((input_batch, input_ids), 0)\n",
        "           input_batch[len(input_batch)-1][j] = k\n",
        "           word_scores_batch.append(word_scores[k].item())\n",
        "      \n",
        "      #probability_input calculates the p(label|sentence) of the target model given each masked input sentence\n",
        "      probability_input = compute_probability2(model, input_batch, attention_masks, label)\n",
        "      \n",
        "      m = torch.dot(torch.tensor(word_scores_batch).to(device), probability_input)\n",
        "      logodds_input = math.log(probability_input[0] / (1-probability_input[0]))\n",
        "      logodds_m = math.log(m / (1-m))\n",
        "      woe.append(logodds_input-logodds_m)\n",
        "  return woe\n"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "68yzg0lNIGj4"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qcDFTbBymIk"
      },
      "source": [
        "def input_marg(model): \n",
        "  test_data = loaddata()\n",
        "  device = \"cuda\" if next(model.parameters()).is_cuda else \"cpu\"\n",
        "  iter_data = iter(test_data)\n",
        "  results = []\n",
        "\n",
        "  for i in range(SAMPLE_SIZE):\n",
        "    nextsample = next(iter_data)\n",
        "    inputsequences = nextsample[0].to(device)\n",
        "    inputmask =  nextsample[1].to(device)\n",
        "    labels = nextsample[2].to(device)\n",
        "    print(\"\")\n",
        "    print(labels)\n",
        "    token_ids = tokenizer.convert_ids_to_tokens(inputsequences[0][1:20])\n",
        "    label = torch.unsqueeze(labels[0],0)\n",
        "    woe = calculate_woe(model, torch.unsqueeze(inputsequences[0][1:20],0),torch.unsqueeze(inputmask[0][1:20],0),  label, SIGMA)\n",
        "    results.append((label,woe, token_ids))\n",
        "  return results\n",
        "      "
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fXG0ScGmyviL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e7844675-9b7c-4641-8542-f4d6323664a9"
      },
      "source": [
        "cnnresults = input_marg(cnn)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6,919 training samples.\n",
            "  876 validation samples.\n",
            "1,822 test samples.\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([1], device='cuda:0')\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([1], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tvauU8TX3z4w",
        "outputId": "9b7455d2-926e-4914-e935-260c912577ca"
      },
      "source": [
        "lstmresults = input_marg(lstm)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6,919 training samples.\n",
            "  876 validation samples.\n",
            "1,822 test samples.\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([1], device='cuda:0')\n",
            "\n",
            "tensor([0], device='cuda:0')\n",
            "\n",
            "tensor([1], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9bRSxoKvfub5"
      },
      "source": [
        "#bertresults = input_marg(bert)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KSOQdSZUgtjj",
        "outputId": "67893252-664b-45cf-c0eb-3fa40f276763"
      },
      "source": [
        "for s in cnnresults:\n",
        "  print(\"\")\n",
        "  print(s)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "(tensor([0], device='cuda:0'), [3.4610748016280053, 2.38397474852603, 2.3923195258668084, 1.7607884267928884, 0.3674926114160879, 1.5414679254725554, 3.1376562405806236, 0.1259094629247639, 2.3533538663678235, 1.3535274124980452, 2.104138825536487, 2.059403313810178, 0.3496032190855063, 2.0087733035546256, 1.8335877501232627, 2.165256869315688, 2.0007512542773425, 0.3124485696755457, 1.4257422518866631], ['a', 'cum', '##bers', '##ome', 'and', 'cl', '##iche', '-', 'ridden', 'movie', 'grease', '##d', 'with', 'every', 'emotional', 'device', 'known', 'to', 'man'])\n",
            "\n",
            "(tensor([1], device='cuda:0'), [0.019363717935888314, -0.061479726320976535, -0.00011158547597389656, 0.0010579135350781144, -0.5096246861903642, -0.83539128521819, 0.00045235341203397894, -0.98840695155201, -0.7205186279995546, 0.0007020201621914524, 0.0004937733128724808, -0.0056011366841750565, -0.052475071732195566, -0.06809665905804474, -0.3865993082092616, 0.09922096857926999, -0.0063797004028489646, 0.02325916561745256, 0.00120062864300019], ['it', 'helps', 'that', 'the', 'central', 'performers', 'are', 'experienced', 'actors', ',', 'and', 'that', 'they', 'know', 'their', 'roles', 'so', 'well', '.'])\n",
            "\n",
            "(tensor([0], device='cuda:0'), [1.1593857800076681, 0.026039629497411343, -0.12059176414425271, 0.047466656616150626, -0.0019602993891629783, -0.06639135697456902, -0.00012635308714875615, 0.0004862539243344344, 0.004769082044040196, 0.0002478646134134177, 0.0001485321333016265, -0.00026621084055666877, 0.0016587335763504907, 2.1508498546318755e-05, 0.0018351812269450374, 0.00024672374138789976, 0.2956374193453004, 0.2274537675440501, 0.13785160587558098], ['maris', '##a', 'tome', '##i', 'is', 'good', ',', 'but', 'just', 'a', 'kiss', 'is', 'just', 'a', 'mess', '.', '[SEP]', '[PAD]', '[PAD]'])\n",
            "\n",
            "(tensor([0], device='cuda:0'), [1.0510061928837218, 0.09045357324496783, 0.010407173932442948, -0.12431853656669523, 1.973732190463373e-05, -0.48083609731347754, -0.13185297051285683, 0.00017350439844610044, -0.0005073128334636134, -0.00011091650297956335, 0.009164960712088055, -0.11062804503256074, 1.1842283287322175e-06, 0.08359046667479286, -0.4152135960851655, 0.000985664307944667, 0.0020914591685927686, 1.3520023475654952e-05, 0.20971865266723522], ['the', 'kids', 'often', 'appear', 'to', 'be', 'reading', 'the', 'lines', 'and', 'are', 'incapable', 'of', 'convey', '##ing', 'any', 'emotion', '.', '[SEP]'])\n",
            "\n",
            "(tensor([1], device='cuda:0'), [0.7723314983447738, 0.12020404067544987, 0.04463446672291882, 0.012076127876509357, 0.22656476251532398, 1.1855901874440287, 1.359250788542433, 0.00915294272594247, 1.5651898996744686, 0.13679913198247862, 2.409901004778564, 0.10740964116706664, 0.1325222497626628, 0.09238313635224449, 1.7801834103411265, 0.00730322324296484, 0.006659795644217592, 0.22625325919898343, 0.06395353565273698], ['.', '.', '.', 'there', 'are', 'enough', 'moments', 'of', 'heartbreak', '##ing', 'honesty', 'to', 'keep', 'one', 'glued', 'to', 'the', 'screen', '.'])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J_pLvuDTh1no",
        "outputId": "6f224249-315e-435b-e9d5-f492a84ca0a5"
      },
      "source": [
        "for s in lstmresults:\n",
        "  print(\"\")\n",
        "  print(s)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "(tensor([0], device='cuda:0'), [0.37156821811137064, 0.017492801567271066, 0.013098964308289385, -0.12523953950215794, 0.6860030198666149, 0.0040648343216771465, 0.01789189489033749, 0.22077135663777514, 0.0012861338516603205, 0.3946944524937588, 0.006771245770374801, 0.7494196675600019, 0.003535269832497856, 0.889286613006903, -0.0007121696756314799, -0.15560619923878383, 0.8239208374883507, -0.08432024024142892, 0.5903470888925955], ['what', \"'\", 's', 'at', 'stake', 'in', 'this', 'film', 'is', 'nothing', 'more', 'than', 'an', 'obsolete', ',', 'if', 'irritating', ',', 'notion'])\n",
            "\n",
            "(tensor([0], device='cuda:0'), [0.40896555142323227, -0.5955950919284161, 0.3057715897392918, -0.04506092700470665, 0.001421766975496841, 0.06626859221042797, 0.4055718487843695, -0.05357000897940517, -0.3228402221788481, 0.426460677970432, 0.15528935255175463, 0.04751550712606961, -0.0020060389923851063, 0.6506775834303204, -0.003255061313046259, 0.11698716214179333, 0.8466005020088545, 0.001986620493516278, -0.09617681711339454], ['focuses', 'on', 'joan', \"'\", 's', 'raging', 'hormones', 'and', 'sl', '##edge', '##hammer', '##s', 'the', 'audience', 'with', 'spanish', 'inquisition', '##s', 'about'])\n",
            "\n",
            "(tensor([1], device='cuda:0'), [1.16103182297719, 0.0012977242714224263, 0.000440340276045878, 0.8525162157831951, 0.8733406704436278, 0.0003850397752544199, -0.017602260428135286, 0.0024655853458970395, 0.032784749304270354, 0.002740932376777583, 0.00029445470778904914, 0.030012541530583248, 0.0117321444802823, 0.04554152532423991, 0.004167907247441635, -0.1936194035136224, 0.21394539771528964, 0.0017867457599752279, 0.002790853145475211], ['it', \"'\", 's', 'end', '##earing', 'to', 'hear', 'madame', 'd', '.', 'refer', 'to', 'her', 'husband', 'as', '`', 'jackie', \"'\", '-'])\n",
            "\n",
            "(tensor([0], device='cuda:0'), [0.9650586225543154, 0.0023952714542503917, 0.0001275163737577134, 0.001520851586297861, 0.00011403158687309234, 0.018450273807231715, 0.05294023310137261, 0.0008059605222588417, 8.719945539403806e-05, -0.0009433075890549203, 0.001838478414058431, 4.859424215047925e-06, 0.00011711185211679687, 3.682270436350388e-05, 0.00014586145193459799, 7.713759170202561e-05, 0.0006567154973098066, 0.3223159265977729, 0.3142237417066571], ['usually', 'when', 'i', 'get', 'this', 'much', 'syrup', ',', 'i', 'like', 'pancakes', 'to', 'go', 'with', 'it', '.', '[SEP]', '[PAD]', '[PAD]'])\n",
            "\n",
            "(tensor([1], device='cuda:0'), [1.3854880798444462, 1.213602179438296, 0.7021498547035558, 0.001648177936168338, 0.026616922696860446, 0.004105427143594298, 0.9412368792914101, 1.5539073486720556, 0.22305246889232966, 1.4805789944283494, 1.4189675708802152, 0.6749713245756142, 0.19030391108981193, 0.6170063098256207, 0.2001068154414123, 0.9941930652124644, 0.08923220691473421, -0.17754423511889872, 0.17628649279112985], ['-', 'l', '##rb', '-', 'a', '-', 'rr', '##b', '-', 'hollywood', 'sheen', 'bed', '##ev', '##ils', 'the', 'film', 'from', 'the', 'very'])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-siGBjIh41m"
      },
      "source": [
        "lstmsentences_pos = [\n",
        "  lstmresults[2], lstmresults[4]\n",
        "]"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EfFiSPjHuXdf"
      },
      "source": [
        "cnnsentences_pos = [\n",
        "  cnnresults[1], cnnresults[4]\n",
        "]"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3egfqDavudON"
      },
      "source": [
        "lstmsentences_neg = [\n",
        "  lstmresults[0], lstmresults[1]\n",
        "]"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UvquUGwFudVr"
      },
      "source": [
        "cnnsentences_neg = [\n",
        "  cnnresults[0], cnnresults[3]\n",
        "]"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVgb-NzM3r5P"
      },
      "source": [
        "bertsentences = [\n",
        "\"it's a lovely film with lovely performances by buy and accorsi .\",\n",
        "\"more romantic , more emotional and ultimately more satisfying than the teary-eyed original .\",\n",
        "\"it's a bit disappointing that it only manages to be decent instead of dead brilliant .\",\n",
        "\"suffers from the lack of a compelling or comprehensible narrative .\"\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6NSVe17e3_kV",
        "outputId": "0a87baf0-c0b9-47f6-8e90-4c4ea66efb40"
      },
      "source": [
        "# LSTM pos\n",
        "for labels, auclist, tokens in lstmsentences_pos:\n",
        "  colored_sentence(lstm, tokens, auclist)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[48;2;255;0;0mit\u001b[0m \u001b[48;2;255;230;234m'\u001b[0m \u001b[48;2;255;230;234ms\u001b[0m \u001b[48;2;225;102;102mend\u001b[0m \u001b[48;2;225;102;102mearing\u001b[0m \u001b[48;2;255;230;234mto\u001b[0m \u001b[48;2;204;229;255mhear\u001b[0m \u001b[48;2;255;230;234mmadame\u001b[0m \u001b[48;2;255;230;234md\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m \u001b[48;2;255;230;234mrefer\u001b[0m \u001b[48;2;255;230;234mto\u001b[0m \u001b[48;2;255;230;234mher\u001b[0m \u001b[48;2;255;230;234mhusband\u001b[0m \u001b[48;2;255;230;234mas\u001b[0m \u001b[48;2;102;178;225m`\u001b[0m \u001b[48;2;255;230;234mjackie\u001b[0m \u001b[48;2;255;230;234m'\u001b[0m \u001b[48;2;255;230;234m-\u001b[0m\n",
            " \u001b[48;2;255;0;0m-\u001b[0m \u001b[48;2;255;0;0ml\u001b[0m \u001b[48;2;225;102;102mrb\u001b[0m \u001b[48;2;255;230;234m-\u001b[0m \u001b[48;2;255;230;234ma\u001b[0m \u001b[48;2;255;230;234m-\u001b[0m \u001b[48;2;225;102;102mrr\u001b[0m \u001b[48;2;255;0;0mb\u001b[0m \u001b[48;2;255;230;234m-\u001b[0m \u001b[48;2;255;0;0mhollywood\u001b[0m \u001b[48;2;255;0;0msheen\u001b[0m \u001b[48;2;225;102;102mbed\u001b[0m \u001b[48;2;255;230;234mev\u001b[0m \u001b[48;2;225;102;102mils\u001b[0m \u001b[48;2;255;230;234mthe\u001b[0m \u001b[48;2;225;102;102mfilm\u001b[0m \u001b[48;2;255;230;234mfrom\u001b[0m \u001b[48;2;102;178;225mthe\u001b[0m \u001b[48;2;255;230;234mvery\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3u8MQtPahV4w",
        "outputId": "87aed984-db7e-47d6-c791-2b912959e756"
      },
      "source": [
        "# CNN pos\n",
        "for labels, auclist, tokens in cnnsentences_pos:\n",
        "  colored_sentence(cnn, tokens, auclist)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[48;2;255;230;234mit\u001b[0m \u001b[48;2;204;229;255mhelps\u001b[0m \u001b[48;2;204;229;255mthat\u001b[0m \u001b[48;2;255;230;234mthe\u001b[0m \u001b[48;2;0;0;255mcentral\u001b[0m \u001b[48;2;0;0;255mperformers\u001b[0m \u001b[48;2;255;230;234mare\u001b[0m \u001b[48;2;0;0;255mexperienced\u001b[0m \u001b[48;2;0;0;255mactors\u001b[0m \u001b[48;2;255;230;234m,\u001b[0m \u001b[48;2;255;230;234mand\u001b[0m \u001b[48;2;204;229;255mthat\u001b[0m \u001b[48;2;204;229;255mthey\u001b[0m \u001b[48;2;204;229;255mknow\u001b[0m \u001b[48;2;0;0;255mtheir\u001b[0m \u001b[48;2;255;230;234mroles\u001b[0m \u001b[48;2;204;229;255mso\u001b[0m \u001b[48;2;255;230;234mwell\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m\n",
            " \u001b[48;2;225;102;102m.\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m \u001b[48;2;255;230;234mthere\u001b[0m \u001b[48;2;255;230;234mare\u001b[0m \u001b[48;2;255;0;0menough\u001b[0m \u001b[48;2;255;0;0mmoments\u001b[0m \u001b[48;2;255;230;234mof\u001b[0m \u001b[48;2;255;0;0mheartbreak\u001b[0m\u001b[48;2;255;230;234ming\u001b[0m \u001b[48;2;255;0;0mhonesty\u001b[0m \u001b[48;2;255;230;234mto\u001b[0m \u001b[48;2;255;230;234mkeep\u001b[0m \u001b[48;2;255;230;234mone\u001b[0m \u001b[48;2;255;0;0mglued\u001b[0m \u001b[48;2;255;230;234mto\u001b[0m \u001b[48;2;255;230;234mthe\u001b[0m \u001b[48;2;255;230;234mscreen\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SSB0ZJLuvMKN",
        "outputId": "676b8d2c-63f0-4f28-d2af-f2809d888ba3"
      },
      "source": [
        "# LSTM neg\n",
        "for labels, auclist, tokens in lstmsentences_neg:\n",
        "  colored_sentence(lstm, tokens, auclist)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[48;2;255;204;204mwhat\u001b[0m \u001b[48;2;255;230;234m'\u001b[0m \u001b[48;2;255;230;234ms\u001b[0m \u001b[48;2;102;178;225mat\u001b[0m \u001b[48;2;225;102;102mstake\u001b[0m \u001b[48;2;255;230;234min\u001b[0m \u001b[48;2;255;230;234mthis\u001b[0m \u001b[48;2;255;230;234mfilm\u001b[0m \u001b[48;2;255;230;234mis\u001b[0m \u001b[48;2;255;204;204mnothing\u001b[0m \u001b[48;2;255;230;234mmore\u001b[0m \u001b[48;2;225;102;102mthan\u001b[0m \u001b[48;2;255;230;234man\u001b[0m \u001b[48;2;225;102;102mobsolete\u001b[0m \u001b[48;2;204;229;255m,\u001b[0m \u001b[48;2;102;178;225mif\u001b[0m \u001b[48;2;225;102;102mirritating\u001b[0m \u001b[48;2;204;229;255m,\u001b[0m \u001b[48;2;225;102;102mnotion\u001b[0m\n",
            " \u001b[48;2;255;204;204mfocuses\u001b[0m \u001b[48;2;0;0;255mon\u001b[0m \u001b[48;2;255;204;204mjoan\u001b[0m \u001b[48;2;204;229;255m'\u001b[0m \u001b[48;2;255;230;234ms\u001b[0m \u001b[48;2;255;230;234mraging\u001b[0m \u001b[48;2;255;204;204mhormones\u001b[0m \u001b[48;2;204;229;255mand\u001b[0m \u001b[48;2;0;0;255msl\u001b[0m \u001b[48;2;255;204;204medge\u001b[0m \u001b[48;2;255;230;234mhammer\u001b[0m \u001b[48;2;255;230;234ms\u001b[0m \u001b[48;2;204;229;255mthe\u001b[0m \u001b[48;2;225;102;102maudience\u001b[0m \u001b[48;2;204;229;255mwith\u001b[0m \u001b[48;2;255;230;234mspanish\u001b[0m \u001b[48;2;225;102;102minquisition\u001b[0m \u001b[48;2;255;230;234ms\u001b[0m \u001b[48;2;204;229;255mabout\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IZs4L7_HvMS-",
        "outputId": "4755b1a3-ee34-4f39-fbfa-07db52c4db39"
      },
      "source": [
        "# CNN neg\n",
        "for labels, auclist, tokens in cnnsentences_neg:\n",
        "  colored_sentence(cnn, tokens, auclist)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;0;0mcum\u001b[0m\u001b[48;2;255;0;0mbers\u001b[0m\u001b[48;2;255;0;0mome\u001b[0m \u001b[48;2;255;204;204mand\u001b[0m \u001b[48;2;255;0;0mcl\u001b[0m\u001b[48;2;255;0;0miche\u001b[0m \u001b[48;2;255;230;234m-\u001b[0m \u001b[48;2;255;0;0mridden\u001b[0m \u001b[48;2;255;0;0mmovie\u001b[0m \u001b[48;2;255;0;0mgrease\u001b[0m\u001b[48;2;255;0;0md\u001b[0m \u001b[48;2;255;204;204mwith\u001b[0m \u001b[48;2;255;0;0mevery\u001b[0m \u001b[48;2;255;0;0memotional\u001b[0m \u001b[48;2;255;0;0mdevice\u001b[0m \u001b[48;2;255;0;0mknown\u001b[0m \u001b[48;2;255;204;204mto\u001b[0m \u001b[48;2;255;0;0mman\u001b[0m\n",
            " \u001b[48;2;255;0;0mthe\u001b[0m \u001b[48;2;255;230;234mkids\u001b[0m \u001b[48;2;255;230;234moften\u001b[0m \u001b[48;2;102;178;225mappear\u001b[0m \u001b[48;2;255;230;234mto\u001b[0m \u001b[48;2;0;0;255mbe\u001b[0m \u001b[48;2;102;178;225mreading\u001b[0m \u001b[48;2;255;230;234mthe\u001b[0m \u001b[48;2;204;229;255mlines\u001b[0m \u001b[48;2;204;229;255mand\u001b[0m \u001b[48;2;255;230;234mare\u001b[0m \u001b[48;2;102;178;225mincapable\u001b[0m \u001b[48;2;255;230;234mof\u001b[0m \u001b[48;2;255;230;234mconvey\u001b[0m\u001b[48;2;0;0;255ming\u001b[0m \u001b[48;2;255;230;234many\u001b[0m \u001b[48;2;255;230;234memotion\u001b[0m \u001b[48;2;255;230;234m.\u001b[0m \u001b[48;2;255;230;234m[SEP]\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcW4-0nD387X"
      },
      "source": [
        "def colored_sentence(model, tokenized_sentence, auclist):\n",
        "    # define some color for different levels of effect\n",
        "    red3 = [255, 0, 0]\n",
        "    red2 = [225, 102, 102]\n",
        "    red1 = [255, 204, 204]\n",
        "    red0 = [255, 230, 234]\n",
        "    blue0 = [204, 229, 255]\n",
        "    blue1 = [204, 229, 255]\n",
        "    blue2 = [102, 178, 225]\n",
        "    blue3 = [0, 0, 255]\n",
        "\n",
        "    splits = [-0.2, -0.1, -0.05,0, 0.3, 0.5, 1]\n",
        "\n",
        "    colored = []\n",
        "    joined = []\n",
        "\n",
        "    for i in range(len(tokenized_sentence)):\n",
        "        if tokenized_sentence[i][0] == '#':\n",
        "          tokenized_sentence[i] = tokenized_sentence[i][2:]\n",
        "          joined.append(1)\n",
        "        else:\n",
        "          joined.append(0)\n",
        "\n",
        "        if auclist[i] > splits[6]:  # very positive\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(red3[0]),\n",
        "                    str(red3[1]),\n",
        "                    str(red3[2]),\n",
        "                    tokenized_sentence[i],\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[5]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(red2[0]), str(red2[1]), str(red2[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[4]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(red1[0]), str(red1[1]), str(red1[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[3]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(red0[0]), str(red0[1]), str(red0[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[2]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(blue0[0]), str(blue0[1]), str(blue0[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[1]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(blue1[0]), str(blue1[1]), str(blue1[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "        elif auclist[i] > splits[0]:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(blue2[0]), str(blue2[1]), str(blue2[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "       \n",
        "        else:\n",
        "            colored.append(\n",
        "                \"\\033[48;2;{};{};{}m{}\\033[0m\".format(\n",
        "                    str(blue3[0]), str(blue3[1]), str(blue3[2]), tokenized_sentence[i]\n",
        "                )\n",
        "            )\n",
        "    sent = \"\"\n",
        "    \n",
        "    for i, elem in enumerate(colored):\n",
        "      if joined[i] == 1:\n",
        "        sent = sent+str(elem)\n",
        "      else:\n",
        "        sent = sent+\" \"+str(elem)\n",
        "\n",
        "    print(sent)"
      ],
      "execution_count": 25,
      "outputs": []
    }
  ]
}