NLP_Week1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_Week1.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/modabbir24/Learning-Content/blob/master/NLP_Week1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pY6tFUzXdkwL",
        "colab_type": "text"
      },
      "source": [
        "lemmatization_demo\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ag-vGuMKdu-R",
        "colab_type": "text"
      },
      "source": [
        "Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJGRif2Pdn_q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "716015c0-808e-4ebb-b7ea-5af8266b0b06"
      },
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dHSH4GLdd6ot",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = \"This is Andrew's text, isn't it?\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T62S1X8GegoR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c585f838-a771-428b-e789-d69532f411f3"
      },
      "source": [
        "tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
        "tokenizer.tokenize(text)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['This', 'is', \"Andrew's\", 'text,', \"isn't\", 'it?']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5Nn9R7wevq8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "569330f4-8604-4e36-95bb-e45f03e311a3"
      },
      "source": [
        "tokenizer = nltk.tokenize.TreebankWordTokenizer()\n",
        "tokenizer.tokenize(text)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['This', 'is', 'Andrew', \"'s\", 'text', ',', 'is', \"n't\", 'it', '?']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q2MUuWbffHpe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b3d09abb-eebc-49ac-ba53-b4cab19af80b"
      },
      "source": [
        "tokenizer = nltk.tokenize.WordPunctTokenizer()\n",
        "tokenizer.tokenize(text)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['This', 'is', 'Andrew', \"'\", 's', 'text', ',', 'isn', \"'\", 't', 'it', '?']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNKX9yiXfU_e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = \"feet wolves cats talked\"\n",
        "tokenizer = nltk.tokenize.TreebankWordTokenizer()\n",
        "tokens = tokenizer.tokenize(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MffjXO6bfvWl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9f29873e-82b2-4b49-c166-e6bcf6210921"
      },
      "source": [
        "stemmer=nltk.stem.PorterStemmer()\n",
        "\" \".join(stemmer.stem(token) for token in tokens)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'feet wolv cat talk'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7L1l-IE4hBY-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "eac4c33b-26e3-4ba2-9a6b-f4eeda11eec6"
      },
      "source": [
        "stemmer = nltk.stem.WordNetLemmatizer()\n",
        "\" \".join(stemmer.lemmatize(token) for token in tokens)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'foot wolf cat talked'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4RbtjuskiDS2",
        "colab_type": "text"
      },
      "source": [
        "tf_idf"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGTElhrFhOBz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "ad94d967-db3c-4392-dcd7-efabf411000d"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "import pandas as pd\n",
        "texts = [\"good movie\", \"not a good movie\", \"did not like\", \n",
        "    \"i like it\", \"good one\"]\n",
        "\n",
        "tfidf = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, 2))\n",
        "features = tfidf.fit_transform(texts)\n",
        "pd.DataFrame(\n",
        "    features.todense(),\n",
        "    columns=tfidf.get_feature_names()\n",
        ")"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>good movie</th>\n",
              "      <th>like</th>\n",
              "      <th>movie</th>\n",
              "      <th>not</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.707107</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.707107</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.577350</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.577350</td>\n",
              "      <td>0.577350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.707107</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.707107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   good movie      like     movie       not\n",
              "0    0.707107  0.000000  0.707107  0.000000\n",
              "1    0.577350  0.000000  0.577350  0.577350\n",
              "2    0.000000  0.707107  0.000000  0.707107\n",
              "3    0.000000  1.000000  0.000000  0.000000\n",
              "4    0.000000  0.000000  0.000000  0.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    }
  ]
}