04_02_Sentiment_Analysis_with_TorchText.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"04_02_Sentiment_Analysis_with_TorchText.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyM1Y66iHpZIpvg6TX6U00ZP"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"Y2fkP_np3Zpl"},"source":["<!--BOOK_INFORMATION-->\n","<img align=\"left\" style=\"width:80px;height:98px;padding-right:20px;\" src=\"https://raw.githubusercontent.com/joe-papa/pytorch-book/main/files/pytorch-book-cover.jpg\">\n","\n","This notebook contains an excerpt from the [PyTorch Pocket Reference](http://pytorchbook.com) book by [Joe Papa](http://joepapa.ai); content is available [on GitHub](https://github.com/joe-papa/pytorch-book)."]},{"cell_type":"markdown","metadata":{"id":"xlA2i0FpGPxe"},"source":["[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/joe-papa/pytorch-book/blob/main/04_02_Sentiment_Analysis_with_TorchText.ipynb)"]},{"cell_type":"markdown","metadata":{"id":"aXkszegKMMMn"},"source":["# Chapter 4 - Neural Network Development Reference Designs\n","# Sentiment Analysis with TorchText"]},{"cell_type":"markdown","metadata":{"id":"fQ5SE1D2mFMs"},"source":["## Data Processing"]},{"cell_type":"code","metadata":{"id":"PAG915JevD3i"},"source":["# For reproducibility\n","import random\n","import torch\n","\n","SEED = 1234\n","torch.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True\n","random_state = random.seed(SEED)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4u0I1d526CxQ","executionInfo":{"status":"ok","timestamp":1615841842114,"user_tz":240,"elapsed":249,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"7b3d42b5-37a0-436d-df45-c1d4eda3c973"},"source":["def generate_bigrams(x):\n","    n_grams = set(zip(*[x[i:] for i in range(2)]))\n","    for n_gram in n_grams:\n","        x.append(' '.join(n_gram))\n","    return x\n","\n","generate_bigrams(['This', 'movie', 'is', 'awesome'])\n","# out:\n","# ['This', 'movie', 'is', 'awesome', 'This movie',\n","#  'movie is', 'is awesome']"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['This', 'movie', 'is', 'awesome', 'This movie', 'movie is', 'is awesome']"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"LLCXyoKT6CxU"},"source":["# import torch\n","# from torchtext import data\n","# from torchtext import datasets\n","\n","# TEXT = data.Field(tokenize = 'spacy',\n","#                   preprocessing = \\\n","#                     generate_bigrams) # <1>\n","\n","# LABEL = data.LabelField(dtype = torch.float) # <2>\n","\n","# train_data, test_data = \\\n","#   datasets.IMDB.splits(TEXT, LABEL) # <3>\n","# train_data, valid_data = train_data.split(\n","#     random_state=random_state) # <4>"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AEjFcr_aJgIr"},"source":["from torchtext.datasets import IMDB\n","from torch.utils.data.dataset import random_split\n","\n","train_iter, test_iter = IMDB(\n","    split=('train', 'test')) #<1>\n","\n","train_dataset = list(train_iter) #<2>\n","test_data = list(test_iter)\n","\n","num_train = int(len(train_dataset) * 0.70)\n","train_data, valid_data = \\\n","    random_split(train_dataset, \n","        [num_train, \n","         len(train_dataset) - num_train]) # <3>"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7IOq1ufhq01Q","executionInfo":{"status":"ok","timestamp":1615841915453,"user_tz":240,"elapsed":357,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"ffa9f08a-66ac-4119-a257-c4fd2c035ff5"},"source":["print(len(train_data), len(valid_data), len(test_data))\n","# out:17500 7500 25000\n","\n","data_index = 21\n","print(train_data[data_index][0])\n","# out: (your results may vary)\n","#   pos\n","\n","print(train_data[data_index][1])\n","# out: (your results may vary)\n","# ['This', 'film', 'moved', 'me', 'beyond', 'comprehension', ..."],"execution_count":null,"outputs":[{"output_type":"stream","text":["17500 7500 25000\n","neg\n","I missed the beginning but I did see most of it. A friend got it on DVD in the cheap room at FYE.<br /><br />The skits are all very short, and yet most of them are still too long. The majority of them, they seem to have forgotten to have something funny! Quite a lot of racist/sexist/\"homophobic\" humor in it, skits based on stereotypes, or skits which use racist terms for people.<br /><br />I'm trying to remember anything I thought was funny in it, and I'm having trouble.... The logo for the Tunnel Vision network is a lipsticked mouth with an eyeball in it. The mouth opens and closes over the eye like eyelids. Kind of creepy.<br /><br />What a disappointment. Most of the actors went on to better things, and it's lucky this bomb didn't hold them back.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Sc9_0aaBODCh","executionInfo":{"status":"ok","timestamp":1615842157475,"user_tz":240,"elapsed":239778,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"85e18e96-9ec4-42fa-fe04-478d5e10a886"},"source":["from torchtext.data.utils import get_tokenizer\n","from collections import Counter\n","from torchtext.vocab import Vocab\n","\n","tokenizer = get_tokenizer('spacy') # <1>\n","counter = Counter()\n","for (label, line) in train_data:\n","    counter.update(generate_bigrams(\n","        tokenizer(line))) # <2>\n","vocab = Vocab(counter, \n","              max_size = 25000, \n","              vectors = \"glove.6B.100d\", \n","              unk_init = torch.Tensor.normal_) # <3>"],"execution_count":null,"outputs":[{"output_type":"stream","text":[".vector_cache/glove.6B.zip: 862MB [02:40, 5.36MB/s]                           \n","100%|█████████▉| 398612/400000 [00:14<00:00, 26958.02it/s]"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wjbOq1dzSF_i","executionInfo":{"status":"ok","timestamp":1615842243764,"user_tz":240,"elapsed":254,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"5e4541bb-69ca-4602-80fc-d0f0d45d045a"},"source":["print(len(counter))\n","print(len(vocab))\n","print(vocab['<pad>'])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["1212925\n","25002\n","1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PBKr3GBUQcGG","executionInfo":{"status":"ok","timestamp":1615842248568,"user_tz":240,"elapsed":293,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"1ffde066-ca73-486f-91ec-9d88d914c94d"},"source":["tokens = generate_bigrams(tokenizer(('this is the greatest movie ever!')))\n","for token in tokens:\n","  print(token, vocab[token]) "],"execution_count":null,"outputs":[{"output_type":"stream","text":["this 16\n","is 9\n","the 2\n","greatest 1457\n","movie 22\n","ever 170\n","! 40\n","this is 224\n","is the 181\n","greatest movie 0\n","movie ever 4749\n","the greatest 2349\n","ever ! 20411\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RLiLLE-tO9NR","executionInfo":{"status":"ok","timestamp":1615842250162,"user_tz":240,"elapsed":240,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"0a21ee4a-0c99-4e76-defc-9fbe539ee6c4"},"source":["text_pipeline = lambda x: [vocab[token] \n","      for token in generate_bigrams(tokenizer(x))]\n","\n","label_pipeline = lambda x: 1 if x=='pos' else 0\n","\n","print(text_pipeline('the movie was horrible'))\n","# out: [2, 22, 19, 942, 157, 14859, 538]\n","\n","print(label_pipeline('neg'))\n","# out: 0"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[2, 22, 19, 942, 14859, 157, 538]\n","0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EMy__zqIPbll"},"source":["from torch.utils.data import DataLoader\n","from torch.nn.utils.rnn import pad_sequence\n","\n","device = torch.device(\"cuda\" if \n","    torch.cuda.is_available() else \"cpu\")\n","\n","def collate_batch(batch):\n","   label_list, text_list = [], []\n","   for (_label, _text) in batch:\n","        label_list.append(label_pipeline(_label))\n","        processed_text = torch.tensor(text_pipeline(_text))\n","        text_list.append(processed_text)\n","   return (torch.tensor(label_list, dtype=torch.float64).to(device), \n","          pad_sequence(text_list, \n","                       padding_value=1.0).to(device))\n","\n","batch_size = 64    \n","def batch_sampler():\n","    indices = [(i, len(tokenizer(s[1]))) for i, s in enumerate(train_dataset)]\n","    random.shuffle(indices)\n","    pooled_indices = []\n","    # create pool of indices with similar lengths \n","    for i in range(0, len(indices), batch_size * 100):\n","        pooled_indices.extend(sorted(indices[i:i + batch_size * 100], key=lambda x: x[1]))\n","\n","    pooled_indices = [x[0] for x in pooled_indices]\n","\n","    # yield indices for current batch\n","    for i in range(0, len(pooled_indices), batch_size):\n","        yield pooled_indices[i:i + batch_size]\n","\n","BATCH_SIZE = 64\n","\n","train_dataloader = DataLoader(train_data,\n","                  # batch_sampler=batch_sampler(),\n","                  collate_fn=collate_batch,\n","                  batch_size=BATCH_SIZE,\n","                  shuffle=True)\n","                  # collate_fn=collate_batch)\n","valid_dataloader = DataLoader(valid_data, \n","                  batch_size=BATCH_SIZE,\n","                  shuffle=True, \n","                  collate_fn=collate_batch)\n","test_dataloader = DataLoader(test_data, \n","                  batch_size=BATCH_SIZE,\n","                  shuffle=True, \n","                  collate_fn=collate_batch)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"39uZzYA2G_oH","executionInfo":{"status":"ok","timestamp":1615843815261,"user_tz":240,"elapsed":355,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"e020de8e-ab01-4382-f076-a07de9ebcb58"},"source":["label, text = next(iter(train_dataloader))\n","print(label.size(), text.size())\n","print(label, text)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["torch.Size([64]) torch.Size([3600, 64])\n","tensor([0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0.,\n","        0., 1., 0., 1., 1., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0.,\n","        1., 0., 0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n","        1., 1., 1., 1., 0., 0., 1., 0., 1., 1.], device='cuda:0',\n","       dtype=torch.float64) tensor([[   68,   316,  2497,  ...,    14,  1917,  1555],\n","        [   24,   153,  7606,  ...,  9193,    27,   345],\n","        [   19,     2, 14628,  ...,     0, 12789,    56],\n","        ...,\n","        [    1,     1,     1,  ...,     1,     1,     1],\n","        [    1,     1,     1,  ...,     1,     1,     1],\n","        [    1,     1,     1,  ...,     1,     1,     1]], device='cuda:0')\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EWQrfaT46CxW"},"source":["# device = torch.device('cuda' \n","#   if torch.cuda.is_available() else 'cpu')\n","\n","# train_iterator, valid_iterator, test_iterator = \\\n","#   data.BucketIterator.splits(\n","#     (train_data, valid_data, test_data), \n","#     batch_size = 64, \n","#     device = device)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rnqlCW9arhLP"},"source":["## Model Design"]},{"cell_type":"code","metadata":{"id":"YcA1qdwD6CxY"},"source":["import torch.nn as nn\n","import torch.nn.functional as F\n","\n","class FastText(nn.Module):\n","    def __init__(self, \n","                 vocab_size, \n","                 embedding_dim, \n","                 output_dim, \n","                 pad_idx):\n","        super().__init__()\n","        self.embedding = nn.Embedding(\n","            vocab_size, \n","            embedding_dim, \n","            padding_idx=pad_idx)\n","        self.fc = nn.Linear(embedding_dim, \n","                            output_dim)\n","        \n","    def forward(self, text):\n","        embedded = self.embedding(text)\n","        embedded = embedded.permute(1, 0, 2)\n","        pooled = F.avg_pool2d(\n","            embedded, \n","            (embedded.shape[1], 1)).squeeze(1) \n","        return self.fc(pooled)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"w43pDDgT6CxY"},"source":["model = FastText(\n","            vocab_size = len(vocab), \n","            embedding_dim = 100, \n","            output_dim = 1, \n","            pad_idx = \\\n","              vocab['<pad>'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"yndecIPP6Cxa"},"source":["Not forgetting to zero the initial weights of our unknown and padding tokens."]},{"cell_type":"code","metadata":{"id":"Rbj-CX7J6Cxb"},"source":["pretrained_embeddings = vocab.vectors # <1>\n","model.embedding.weight.data.copy_(\n","                    pretrained_embeddings) # <2>\n","\n","EMBEDDING_DIM = 100\n","unk_idx = vocab['<unk>'] # <3>\n","pad_idx = vocab['<pad>']\n","model.embedding.weight.data[unk_idx] = \\\n","      torch.zeros(EMBEDDING_DIM)          # <4>\n","model.embedding.weight.data[pad_idx] = \\\n","      torch.zeros(EMBEDDING_DIM)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"cRrHD3Ss6Cxb"},"source":["## Train & Validation"]},{"cell_type":"code","metadata":{"id":"7lpQKr4u-qjA"},"source":["import torch.optim as optim\n","\n","optimizer = optim.Adam(model.parameters())\n","criterion = nn.BCEWithLogitsLoss()\n","\n","model = model.to(device)\n","criterion = criterion.to(device)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"B7hBXR41-4HE","executionInfo":{"status":"ok","timestamp":1615844247885,"user_tz":240,"elapsed":211542,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"8dbc4e39-8490-499d-fa90-654d81f60e64"},"source":["for epoch in range(5):\n","  epoch_loss = 0\n","  epoch_acc = 0\n","  \n","  model.train()\n","  for label, text in train_dataloader:\n","      optimizer.zero_grad()\n","      predictions = model(text).squeeze(1)\n","      loss = criterion(predictions, label)\n","      \n","      rounded_preds = torch.round(\n","          torch.sigmoid(predictions))\n","      correct = \\\n","        (rounded_preds == label).float()\n","      acc = correct.sum() / len(correct)\n","      \n","      loss.backward()\n","      optimizer.step()\n","      epoch_loss += loss.item()\n","      epoch_acc += acc.item()\n","\n","  print(\"Epoch %d Train: Loss: %.4f Acc: %.4f\" %\n","          (epoch,\n","          epoch_loss / len(train_dataloader), \n","          epoch_acc / len(train_dataloader)))\n","\n","  epoch_loss = 0\n","  epoch_acc = 0\n","  model.eval()\n","  with torch.no_grad():\n","    for label, text in valid_dataloader:\n","      predictions = model(text).squeeze(1)\n","      loss = criterion(predictions, label)\n","      \n","      rounded_preds = torch.round(\n","          torch.sigmoid(predictions))\n","      correct = \\\n","        (rounded_preds == label).float()\n","      acc = correct.sum() / len(correct)\n","      \n","      epoch_loss += loss.item()\n","      epoch_acc += acc.item()\n","\n","  print(\"Epoch %d Valid: Loss: %.4f Acc: %.4f\" %\n","          (epoch,\n","          epoch_loss / len(valid_dataloader), \n","          epoch_acc / len(valid_dataloader)))\n","  \n","# out: (your results may vary)\n","# Epoch 0 Train: Loss: 0.6523 Acc: 0.7165\n","# Epoch 0 Valid: Loss: 0.5259 Acc: 0.7474\n","# Epoch 1 Train: Loss: 0.5935 Acc: 0.7765\n","# Epoch 1 Valid: Loss: 0.4571 Acc: 0.7933\n","# Epoch 2 Train: Loss: 0.5230 Acc: 0.8257\n","# Epoch 2 Valid: Loss: 0.4103 Acc: 0.8245\n","# Epoch 3 Train: Loss: 0.4559 Acc: 0.8598\n","# Epoch 3 Valid: Loss: 0.3828 Acc: 0.8549\n","# Epoch 4 Train: Loss: 0.4004 Acc: 0.8813\n","# Epoch 4 Valid: Loss: 0.3781 Acc: 0.8675"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 0 Train: Loss: 0.6468 Acc: 0.7353\n","Epoch 0 Valid: Loss: 0.6184 Acc: 0.7452\n","Epoch 1 Train: Loss: 0.5689 Acc: 0.8041\n","Epoch 1 Valid: Loss: 0.5461 Acc: 0.8002\n","Epoch 2 Train: Loss: 0.4909 Acc: 0.8465\n","Epoch 2 Valid: Loss: 0.4798 Acc: 0.8373\n","Epoch 3 Train: Loss: 0.4259 Acc: 0.8739\n","Epoch 3 Valid: Loss: 0.4307 Acc: 0.8565\n","Epoch 4 Train: Loss: 0.3759 Acc: 0.8878\n","Epoch 4 Valid: Loss: 0.3962 Acc: 0.8644\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"XfBcC4qWDkcS"},"source":["# Testing & Deployment"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WdDAlQlaDP0-","executionInfo":{"status":"ok","timestamp":1615844337404,"user_tz":240,"elapsed":41614,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"5a523532-a9b5-4cc4-8803-84c9ad121358"},"source":["test_loss = 0\n","test_acc = 0\n","model.eval() # <1>\n","with torch.no_grad(): # <1>\n","  for label, text in test_dataloader:\n","    predictions = model(text).squeeze(1)\n","    loss = criterion(predictions, label)\n","    \n","    rounded_preds = torch.round(\n","        torch.sigmoid(predictions))\n","    correct = \\\n","      (rounded_preds == label).float()\n","    acc = correct.sum() / len(correct)\n","\n","    test_loss += loss.item()\n","    test_acc += acc.item()\n","\n","print(\"Test: Loss: %.4f Acc: %.4f\" %\n","        (test_loss / len(test_dataloader), \n","        test_acc / len(test_dataloader)))\n","# out: (your results will vary)\n","#   Test: Loss: 0.3821 Acc: 0.8599"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test: Loss: 0.3916 Acc: 0.8648\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"yncFrpAcD2GY","executionInfo":{"status":"ok","timestamp":1615844842113,"user_tz":240,"elapsed":872,"user":{"displayName":"Joe Papa","photoUrl":"","userId":"00487850786587503652"}},"outputId":"fbd7ff12-c2ce-410c-e8d2-88dfdbb80487"},"source":["import spacy\n","nlp = spacy.load('en_core_web_sm')\n","\n","def predict_sentiment(model, sentence):\n","    model.eval()\n","    text = torch.tensor(text_pipeline(sentence)).unsqueeze(1).to(device)\n","    prediction = torch.sigmoid(model(text))\n","    return prediction.item()\n","\n","sentiment = predict_sentiment(model, \n","                  \"Don't waste your time\")\n","print(sentiment)\n","# out: 4.763594888613835e-34\n","\n","sentiment = predict_sentiment(model, \n","                  \"You gotta see this movie!\")\n","print(sentiment)\n","# out: 0.941755473613739"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0.0\n","0.9995007514953613\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"adVgHt83EGbs"},"source":["torch.save(model.state_dict(), 'fasttext-model.pt')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"pJ2CmUZq21-6"},"source":[""],"execution_count":null,"outputs":[]}]}