nlp/pytorch_improved_lstm_bidirectional_and_2_layer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch improved lstm bidirectional and 2 layer",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experiments/blob/master/nlp/pytorch_improved_lstm_bidirectional_and_2_layer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kub0k_oNFN2P",
        "colab_type": "text"
      },
      "source": [
        "using spacy for token\n",
        "\n",
        "include_lengths=True. this basically sets set length of the string as well when we fetch the batch data. Hence it will return tuple of actual vector / length.\n",
        "\n",
        "this is mainly done so that in our NN module we have length's and we can pad/pack the sequence \n",
        "\n",
        "most code taken from https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/2%20-%20Upgraded%20Sentiment%20Analysis.ipynb\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Uwe4kW2Eu0K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "TEXT = data.Field(tokenize = 'spacy', include_lengths = True)\n",
        "LABEL = data.LabelField(dtype = torch.float)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uBAxGCVJGZqY",
        "colab_type": "code",
        "outputId": "9f66ccd5-6e8a-43c6-a5d1-a327eb07abbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from torchtext import datasets\n",
        "\n",
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading aclImdb_v1.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:07<00:00, 11.3MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tov0dkWiGn0G",
        "colab_type": "code",
        "outputId": "bc4b406b-0710-4898-b972-b4b3d2454894",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "import random\n",
        "\n",
        "train_data, valid_data = train_data.split(random_state = random.seed(SEED))\n",
        "\n",
        "\n",
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')\n",
        "\n",
        "print(vars(train_data.examples[0]))\n",
        "\n",
        "train_data, valid_data = train_data.split(random_state = random.seed(SEED))\n",
        "\n",
        "\n",
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of validation examples: {len(valid_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples: 17500\n",
            "Number of testing examples: 25000\n",
            "{'text': ['I', 'very', 'nearly', 'walked', 'out', ',', 'but', 'I', \"'d\", 'paid', 'my', 'money', ',', 'and', 'my', 'nearly', '-', 'as', '-', 'disgusted', 'friend', 'wanted', 'to', 'hold', 'out', '.', 'After', 'the', 'endearing', ',', 'wide', '-', 'eyed', 'innocence', 'of', '\"', 'A', 'New', 'Hope', '\"', 'and', 'the', 'thrilling', 'sophistication', 'of', '\"', 'The', 'Empire', 'Strikes', 'Back', ',', '\"', 'I', 'remember', 'awaiting', '\"', 'Return', 'of', 'the', 'Jedi', '\"', 'with', 'almost', 'aching', 'anticipation', '.', 'But', 'from', 'the', 'opening', 'scene', 'of', 'this', 'insultingly', 'commercial', 'sewage', ',', 'I', 'was', 'bitterly', 'disappointed', ',', 'and', 'enraged', 'at', 'Lucas', '.', 'He', 'should', 'have', 'been', 'ashamed', 'of', 'himself', ',', 'but', 'this', 'abomination', 'undeniably', 'proves', 'that', 'he', 'does', \"n't\", 'have', 'subatomic', 'particle', 'of', 'shame', 'in', 'his', 'cold', ',', 'greedy', 'heart', '.', 'Episode', 'I', 'would', 'go', 'on', 'to', 'reinforce', 'this', 'fact', '--', 'your', 'honor', ',', 'I', 'call', 'Jarjar', 'Binks', '(', 'but', 'please', 'issue', 'barf', 'bags', 'to', 'the', 'members', 'of', 'the', 'jury', 'first).<br', '/><br', '/>From', 'the', 'initial', 'raising', 'of', 'the', 'gate', 'at', 'Jabba', \"'s\", 'lair', ',', 'this', '\"', 'film', '\"', 'was', 'nothing', 'more', 'than', 'a', 'two', '-', 'plus', '-', 'hour', 'commercial', 'for', 'as', 'many', 'licensable', ',', 'profit', '-', 'making', 'action', 'figures', 'as', 'Lucas', 'could', 'cram', 'into', 'it', '--', 'the', 'pig', '-', 'like', 'guards', ',', 'the', 'hokey', 'flesh', '-', 'pigtailed', 'flunky', ',', 'that', 'vile', 'muppet', '-', 'pet', 'of', 'Jabba', \"'s\", ',', 'the', 'new', 'and', 'recycled', 'cabaret', 'figures', ',', 'the', 'monsters', ',', 'etc', '.', ',', 'etc', '.', ',', 'ad', 'vomitum', '.', 'Then', 'there', 'were', 'the', 'detestably', 'cute', 'and', 'marketable', 'Ewoks', '.', 'Pile', 'on', 'top', 'of', 'that', 'all', 'of', 'the', 'rebel', 'alliance', 'aliens', '.', 'Fifteen', 'seconds', 'each', 'on', '-', 'screen', '(', 'or', 'less', ')', 'and', 'the', 'kiddies', 'just', 'GOTTA', 'have', 'one', 'for', 'their', 'collection', '.', 'The', 'blatant', ',', 'exploitative', 'financial', 'baiting', 'of', 'children', 'is', 'nauseating.<br', '/><br', '/>Lucas', 'did', \"n't\", 'even', 'bother', 'to', 'come', 'up', 'with', 'a', 'new', 'plot', '--', 'he', 'just', 'exhumed', 'the', 'Death', 'Star', 'from', '\"', 'A', 'New', 'Hope', '\"', 'and', 'heaved', 'in', 'a', 'boatload', 'of', 'cheap', 'sentiment', '.', 'What', 'an', 'appalling', 'slap', 'in', 'the', 'face', 'to', 'his', 'fans', '.', 'I', 'ca', \"n't\", 'shake', 'the', 'notion', 'that', 'Lucas', 'took', 'a', 'perverse', 'pleasure', 'in', 'inflicting', 'this', 'dreck', 'on', 'his', 'fans', ':', '\"', 'I', \"'ve\", 'got', 'these', 'lemmings', 'hooked', 'so', 'bad', 'that', 'I', 'can', 'crank', 'out', 'the', 'worst', 'piece', 'of', 'stinking', ',', 'putrid', 'garbage', 'that', 'I', 'could', 'dream', 'up', ',', 'and', 'they', \"'ll\", 'flock', 'to', 'the', 'theaters', 'to', 'scarf', 'it', 'up', '.', 'Plus', ',', 'all', 'the', 'kiddies', 'will', 'whine', 'and', 'torture', 'their', 'parents', 'until', 'they', 'buy', 'the', 'brats', 'a', 'complete', 'collection', 'of', 'action', 'figures', 'of', 'every', 'single', 'incidental', 'undeveloped', ',', 'cartoonish', 'caricature', 'that', 'I', 'stuffed', 'in', ',', 'and', 'I', 'get', 'a', 'cut', 'from', 'every', 'single', 'one', '.', 'It', \"'ll\", 'make', 'me', 'even', 'more', 'obscenely', 'rich.\"<br', '/><br', '/>There', 'may', 'have', 'been', 'a', 'paltry', ',', 'partial', 'handful', 'of', 'redeeming', 'moments', 'in', 'this', 'miserable', 'rip', '-', 'off', '.', 'I', 'seem', 'to', 'recall', 'that', 'Harrison', 'Ford', 'managed', 'to', 'just', 'barely', 'keep', 'his', 'nose', 'above', 'the', 'surface', 'of', 'this', 'cesspool', '.', 'But', 'whatever', 'tiny', 'few', 'bright', 'spots', 'there', 'may', 'be', 'are', 'massively', 'obliterated', 'by', 'the', 'offensive', 'commercialism', 'that', 'Lucas', 'so', 'avariciously', 'embraced', 'in', 'this', 'total', ',', 'absolute', 'sell', '-', 'out', 'to', 'profit', '.'], 'label': 'neg'}\n",
            "Number of training examples: 12250\n",
            "Number of validation examples: 5250\n",
            "Number of testing examples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plC-VposHoYd",
        "colab_type": "code",
        "outputId": "54d680f5-5254-4213-e437-c1819b4d707b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from torchtext.vocab import Vectors, GloVe\n",
        "\n",
        "\n",
        "TEXT.build_vocab(train_data, vectors=GloVe(name='6B', dim=300))\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".vector_cache/glove.6B.zip: 862MB [06:31, 2.20MB/s]                           \n",
            "100%|█████████▉| 399904/400000 [00:45<00:00, 8602.59it/s]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w0iqoAvuNeic",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    sort_within_batch = True,\n",
        "    device = device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L64aOnNmNypo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class RNN(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
        "                 bidirectional, dropout, pad_idx, weights):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)\n",
        "\n",
        "        self.embedding.weight = nn.Parameter(weights, requires_grad=False)\n",
        "        \n",
        "        self.rnn = nn.LSTM(embedding_dim, \n",
        "                           hidden_dim, \n",
        "                           num_layers=n_layers, \n",
        "                           bidirectional=bidirectional, \n",
        "                           dropout=dropout)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, text, text_lengths):\n",
        "        \n",
        "        #text = [sent len, batch size]\n",
        "        \n",
        "        embedded = self.dropout(self.embedding(text))\n",
        "        \n",
        "        #embedded = [sent len, batch size, emb dim]\n",
        "        \n",
        "        #pack sequence\n",
        "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths)\n",
        "        \n",
        "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
        "        \n",
        "        #unpack sequence\n",
        "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
        "\n",
        "        #output = [sent len, batch size, hid dim * num directions]\n",
        "        #output over padding tokens are zero tensors\n",
        "        \n",
        "        #hidden = [num layers * num directions, batch size, hid dim]\n",
        "        #cell = [num layers * num directions, batch size, hid dim]\n",
        "        \n",
        "        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers\n",
        "        #and apply dropout\n",
        "        \n",
        "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
        "                \n",
        "        #hidden = [batch size, hid dim * num directions]\n",
        "            \n",
        "        return self.fc(hidden)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HdKw4RmMPlCN",
        "colab_type": "code",
        "outputId": "b073f601-9bfd-4031-cab4-fa66a7b1b933",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "INPUT_DIM = len(TEXT.vocab)\n",
        "EMBEDDING_DIM = 300\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1\n",
        "N_LAYERS = 2\n",
        "BIDIRECTIONAL = True\n",
        "DROPOUT = 0.5\n",
        "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
        "\n",
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "\n",
        "print(pretrained_embeddings.shape)\n",
        "\n",
        "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "\n",
        "\n",
        "\n",
        "model = RNN(INPUT_DIM, \n",
        "            EMBEDDING_DIM, \n",
        "            HIDDEN_DIM, \n",
        "            OUTPUT_DIM, \n",
        "            N_LAYERS, \n",
        "            BIDIRECTIONAL, \n",
        "            DROPOUT, \n",
        "            PAD_IDX,\n",
        "            TEXT.vocab.vectors)\n",
        "\n",
        "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
        "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([84900, 300])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H5jo4ZL1P2LZ",
        "colab_type": "code",
        "outputId": "c5630bb0-4e46-46a2-bada-617fcc2877cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 2,720,257 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ptXZs5veGzIT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "optimizer = optim.Adam(model.parameters())\n",
        "\n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "\n",
        "model = model.to(device)\n",
        "criterion = criterion.to(device)\n",
        "\n",
        "\n",
        "def binary_accuracy(preds, y):\n",
        "    \"\"\"\n",
        "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
        "    \"\"\"\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mee-dmixWosJ",
        "colab_type": "code",
        "outputId": "dc3be190-8fef-400d-9ebe-44ea31030da2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        text, text_lengths = batch.text\n",
        "        \n",
        "        text = text.to(device)\n",
        "        text_lengths = text_lengths.to(device)\n",
        "        \n",
        "        predictions = model(text, text_lengths).squeeze(1)\n",
        "        \n",
        "        loss = criterion(predictions, batch.label)\n",
        "        \n",
        "        acc = binary_accuracy(predictions, batch.label)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "train(model, train_iterator, optimizer, criterion)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r100%|█████████▉| 399904/400000 [01:00<00:00, 8602.59it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.6341418121010065, 0.6328375401596228)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "btzxeKvMWt1G",
        "colab_type": "code",
        "outputId": "80e44f28-bcf5-4754-cac6-e2b30893d668",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text, text_lengths = batch.text\n",
        "            \n",
        "            predictions = model(text, text_lengths).squeeze(1)\n",
        "            \n",
        "            loss = criterion(predictions, batch.label)\n",
        "            \n",
        "            acc = binary_accuracy(predictions, batch.label)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "evaluate(model, valid_iterator, criterion)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.6246603389820421, 0.6869352409638554)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "18aHv1zDWvtB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oVPnNiqhWygA",
        "colab_type": "code",
        "outputId": "fe42d6ab-c2fd-4a5b-d6c7-16cdd022a320",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        }
      },
      "source": [
        "N_EPOCHS = 5\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "    \n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'tut2-model.pt')\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Epoch Time: 0m 28s\n",
            "\tTrain Loss: 0.595 | Train Acc: 68.77%\n",
            "\t Val. Loss: 0.483 |  Val. Acc: 77.80%\n",
            "Epoch: 02 | Epoch Time: 0m 28s\n",
            "\tTrain Loss: 0.532 | Train Acc: 73.13%\n",
            "\t Val. Loss: 0.480 |  Val. Acc: 76.90%\n",
            "Epoch: 03 | Epoch Time: 0m 28s\n",
            "\tTrain Loss: 0.549 | Train Acc: 73.10%\n",
            "\t Val. Loss: 0.507 |  Val. Acc: 75.02%\n",
            "Epoch: 04 | Epoch Time: 0m 28s\n",
            "\tTrain Loss: 0.585 | Train Acc: 68.90%\n",
            "\t Val. Loss: 0.583 |  Val. Acc: 70.05%\n",
            "Epoch: 05 | Epoch Time: 0m 28s\n",
            "\tTrain Loss: 0.469 | Train Acc: 78.26%\n",
            "\t Val. Loss: 0.359 |  Val. Acc: 85.35%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gxm_5hEwDKTN",
        "colab_type": "text"
      },
      "source": [
        "better accuracy "
      ]
    }
  ]
}