RF_mimic_BERT_grid.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RF_mimic_BERT_grid.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ICLUa6PDB1VV",
        "mpFhRc-jOwyA",
        "N4wr6NJ2z1H1",
        "113BH-shPm2J"
      ],
      "authorship_tag": "ABX9TyO2jbSChDkwWuWscL+65+7k",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lupis30puc/BERT_interpretation_with_RF/blob/main/RF_mimic_BERT_grid.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qx--WzAEAMvz"
      },
      "source": [
        "## Set Up"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3d5Y_yn7_owl",
        "outputId": "ec702d8e-a165-40df-efcb-eb9da8bd4087"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Olhw4Uf_qSt"
      },
      "source": [
        "import pandas as pd\r\n",
        "import numpy as np\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "import seaborn as sns\r\n",
        "%matplotlib inline\r\n",
        "\r\n",
        "from sklearn.model_selection import cross_val_score, GridSearchCV\r\n",
        "from sklearn.metrics import classification_report\r\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, roc_curve\r\n",
        "from sklearn.ensemble import RandomForestClassifier\r\n",
        "\r\n",
        "import time\r\n",
        "import joblib\r\n",
        "import torch\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4lAJ1WldAJai"
      },
      "source": [
        "## Initializing the values for the model\n",
        "x_train, x_test, y_train, y_test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LJXHvtRlAJ_S"
      },
      "source": [
        "x_train = pd.read_pickle('/content/drive/MyDrive/Yelp/model_128_/binary_tr_words')\r\n",
        "x_test = pd.read_pickle('/content/drive/MyDrive/Yelp/model_128_/binary_ts_words')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7lE3CO56APFx"
      },
      "source": [
        "y_train = torch.load('/content/drive/MyDrive/Yelp/model_128_/pred_labels_train')\r\n",
        "y_test = torch.load('/content/drive/MyDrive/Yelp/model_128_/pred_labels_test')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICLUa6PDB1VV"
      },
      "source": [
        "## Grid search\n",
        "I performed a grid search to determined the best parameters for the Random Forest model, with a cross validation of 10. \n",
        "\n",
        "Taking into account only the number of estimators (trees) and the minimum sample leaves for each category.\n",
        "\n",
        "It takes 5h 40 min min aprox."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzeVsHtWk99P"
      },
      "source": [
        "param_grid = {'min_samples_leaf': [2, 3], 'n_estimators': [600, 700, 800, 900] }\n",
        "# The parameter grid to explore, as a dictionary mapping estimator parameters to sequences of allowed values."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8mD-ZGXkwXM"
      },
      "source": [
        "rmfr = RandomForestClassifier(random_state=42)\n",
        "grid_search = GridSearchCV(estimator = rmfr, param_grid = param_grid, cv = 10)\n",
        "# Exhaustive search over specified parameter values for an estimator.\n",
        "# The parameters of the estimator used to apply these methods are optimized by cross-validated grid-search over a parameter grid."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "qHcB5u0Q9h72",
        "outputId": "8117af83-0f13-460b-b2df-b5506efd6cb3"
      },
      "source": [
        "%%time\n",
        "grid_search.fit(x_train, y_train) # Run fit with all sets of parameters.\n",
        "#rmfr.fit(x_train,y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3h 59min 58s, sys: 3.46 s, total: 4h 1s\n",
            "Wall time: 4h 37s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GridSearchCV(cv=10, error_score=nan,\n",
              "             estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
              "                                              class_weight=None,\n",
              "                                              criterion='gini', max_depth=None,\n",
              "                                              max_features='auto',\n",
              "                                              max_leaf_nodes=None,\n",
              "                                              max_samples=None,\n",
              "                                              min_impurity_decrease=0.0,\n",
              "                                              min_impurity_split=None,\n",
              "                                              min_samples_leaf=1,\n",
              "                                              min_samples_split=2,\n",
              "                                              min_weight_fraction_leaf=0.0,\n",
              "                                              n_estimators=100, n_jobs=None,\n",
              "                                              oob_score=False, random_state=42,\n",
              "                                              verbose=0, warm_start=False),\n",
              "             iid='deprecated', n_jobs=None,\n",
              "             param_grid={'min_samples_leaf': [2, 3],\n",
              "                         'n_estimators': [600, 700, 800, 900]},\n",
              "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
              "             scoring=None, verbose=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-0TPPm2qlB1u",
        "outputId": "23416b4c-dbbf-43f3-eaf0-03927839a39a"
      },
      "source": [
        "best_model = grid_search.best_params_  # Parameter setting that gave the best results on the hold out data.\n",
        "print(best_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'min_samples_leaf': 2, 'n_estimators': 700}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxMqOAE1fVD8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d3ce02d4-829c-4817-cc34-b924a62e6b3a"
      },
      "source": [
        "# saving the grid search\n",
        "joblib.dump(grid_search, \"/content/drive/MyDrive/Yelp/model_128_/rf_grid_above_400.joblib\", compress=3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/drive/MyDrive/Yelp/model_128_/rf_grid_above_400.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovPhyM9k6scD"
      },
      "source": [
        "rf_700 = joblib.load(\"/content/drive/MyDrive/Yelp/model_128_/rf_grid_above_400.joblib\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kUE3sLBtDyLF",
        "outputId": "231d4779-c7f3-4a83-b4d1-1d822f9ba2cb"
      },
      "source": [
        "#rf_grid.best_params_ \n",
        "print(\"Random Forest 700 estimators, min leaf 2\")\n",
        "predrf_grid = rf_700.predict(x_test)\n",
        "print(\"Confusion Matrix for Random Forest Classifier:\")\n",
        "print(confusion_matrix(y_test,predrf_grid))\n",
        "print(\"Score:\",round(accuracy_score(y_test,predrf_grid)*100,2))\n",
        "print(\"Classification Report:\")\n",
        "print(classification_report(y_test,predrf_grid))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 700 estimators, min leaf 2\n",
            "Confusion Matrix for Random Forest Classifier:\n",
            "[[1163  113]\n",
            " [ 163 1160]]\n",
            "Score: 89.38\n",
            "Classification Report:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.91      0.89      1276\n",
            "           1       0.91      0.88      0.89      1323\n",
            "\n",
            "    accuracy                           0.89      2599\n",
            "   macro avg       0.89      0.89      0.89      2599\n",
            "weighted avg       0.89      0.89      0.89      2599\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4OaVA9ZlWdh"
      },
      "source": [
        "#getting the best model\n",
        "random_f = grid_search.best_estimator_ "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6qojKHlBp3OQ",
        "outputId": "736c37ee-2cd8-4735-fd66-d35c1135f2ba"
      },
      "source": [
        "grid_search.best_score_ "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8961906789072334"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YM8LZPsMiGne",
        "outputId": "9173ac0b-7e6e-4aa8-e1a5-a206b84c2d1c"
      },
      "source": [
        "# for the best estimator on the grid search\n",
        "print(\"Test  Accuracy : %.2f\"%random_f.score(x_test, y_test))\n",
        "print(\"Train Accuracy : %.2f\"%random_f.score(x_train, y_train))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test  Accuracy : 0.89\n",
            "Train Accuracy : 0.98\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "fZ8Rue9F7dqy",
        "outputId": "b3766976-b696-4253-e2dc-889613ea27d2"
      },
      "source": [
        "# saving the grid search\r\n",
        "joblib.dump(grid_search, \"/content/drive/MyDrive/Yelp/model_128_/rf_grid_search_feb_03.joblib\", compress=3)\r\n",
        "\r\n",
        "\r\n",
        "#param_grid = {'min_samples_leaf': [2, 3, 4, 5, 6, 7, 8, 9, 10], 'n_estimators': [200, 300, 1000] }\r\n",
        "\r\n",
        "# best_params = {'min_samples_leaf': 3, 'n_estimators': 1000}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/drive/MyDrive/Yelp/model_128_/rf_grid_search_feb_03.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AfzUeX04Hfg"
      },
      "source": [
        "#make a graph were the test and train accuracy are closer to each other, if they cross or are very close "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mpFhRc-jOwyA"
      },
      "source": [
        "## Initializing the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XCIbuODX_dTe"
      },
      "source": [
        "# load, no need to initialize the loaded_rf\r\n",
        "rf_grid1 = joblib.load(\"/content/drive/MyDrive/Yelp/random_forest/rf_grid_search_feb_03.joblib\")\r\n",
        "rf_grid2 = joblib.load(\"/content/drive/MyDrive/Yelp/random_forest/rf_grid_not_1000.joblib\")\r\n",
        "rf_grid3 = joblib.load(\"/content/drive/MyDrive/Yelp/model_128_/rf_grid_above_400.joblib\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4JA0jvbYhhaI",
        "outputId": "87bab8c7-200d-4c4a-8e14-b2b14f75f4c4"
      },
      "source": [
        "rf_grid1.best_params_  # Parameter setting that gave the best results on the hold out data.\n",
        "print(rf_grid1.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'min_samples_leaf': 3, 'n_estimators': 1000}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HOZz5KUbECUR",
        "outputId": "897f332b-4956-473e-e988-009ed7a8bc4e"
      },
      "source": [
        "rf_grid1.get_params"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method BaseEstimator.get_params of GridSearchCV(cv=10, error_score=nan,\n",
              "             estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
              "                                              class_weight=None,\n",
              "                                              criterion='gini', max_depth=None,\n",
              "                                              max_features='auto',\n",
              "                                              max_leaf_nodes=None,\n",
              "                                              max_samples=None,\n",
              "                                              min_impurity_decrease=0.0,\n",
              "                                              min_impurity_split=None,\n",
              "                                              min_samples_leaf=1,\n",
              "                                              min_samples_split=2,\n",
              "                                              min_weight_fraction_leaf=0.0,\n",
              "                                              n_estimators=100, n_jobs=None,\n",
              "                                              oob_score=False, random_state=42,\n",
              "                                              verbose=0, warm_start=False),\n",
              "             iid='deprecated', n_jobs=None,\n",
              "             param_grid={'min_samples_leaf': [2, 3, 4, 5, 6, 7, 8, 9, 10],\n",
              "                         'n_estimators': [200, 300, 1000]},\n",
              "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
              "             scoring=None, verbose=0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8HBPcemF9Sgd",
        "outputId": "32f22ebf-71d7-4032-b0fe-75d9c2b7a3c7"
      },
      "source": [
        "rf_grid1.cv_results_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'mean_fit_time': array([ 55.54081461,  82.90926821, 273.24089348,  44.40179167,\n",
              "         65.78138323, 218.3534724 ,  38.71027176,  57.88587589,\n",
              "        192.15202942,  35.31426017,  52.37427707, 172.94383399,\n",
              "         32.9204145 ,  49.10529478, 159.11007044,  30.55402312,\n",
              "         45.27990081, 149.44918005,  28.86918001,  42.85899184,\n",
              "        141.48494241,  27.5933332 ,  41.09448581, 135.22420564,\n",
              "         26.13492577,  38.93934672, 128.21310306]),\n",
              " 'mean_score_time': array([0.19013646, 0.26091881, 0.73954639, 0.16870031, 0.22845831,\n",
              "        0.60385084, 0.15839217, 0.20378625, 0.54536035, 0.14600222,\n",
              "        0.19062035, 0.51145914, 0.14231076, 0.1949301 , 0.47538004,\n",
              "        0.13539629, 0.18288703, 0.46241152, 0.13231564, 0.17350538,\n",
              "        0.44282076, 0.12728198, 0.16603389, 0.43349113, 0.12460036,\n",
              "        0.16023617, 0.41210139]),\n",
              " 'mean_test_score': array([0.89224643, 0.89234203, 0.89397775, 0.89292034, 0.89407437,\n",
              "        0.89503656, 0.8900335 , 0.89272701, 0.89234258, 0.89022553,\n",
              "        0.89118809, 0.89061061, 0.88801307, 0.88945547, 0.88974495,\n",
              "        0.88685885, 0.8866658 , 0.88782002, 0.88522275, 0.88580051,\n",
              "        0.88705051, 0.88483749, 0.88445297, 0.88541525, 0.88233712,\n",
              "        0.88358795, 0.88454903]),\n",
              " 'param_min_samples_leaf': masked_array(data=[2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7,\n",
              "                    8, 8, 8, 9, 9, 9, 10, 10, 10],\n",
              "              mask=[False, False, False, False, False, False, False, False,\n",
              "                    False, False, False, False, False, False, False, False,\n",
              "                    False, False, False, False, False, False, False, False,\n",
              "                    False, False, False],\n",
              "        fill_value='?',\n",
              "             dtype=object),\n",
              " 'param_n_estimators': masked_array(data=[200, 300, 1000, 200, 300, 1000, 200, 300, 1000, 200,\n",
              "                    300, 1000, 200, 300, 1000, 200, 300, 1000, 200, 300,\n",
              "                    1000, 200, 300, 1000, 200, 300, 1000],\n",
              "              mask=[False, False, False, False, False, False, False, False,\n",
              "                    False, False, False, False, False, False, False, False,\n",
              "                    False, False, False, False, False, False, False, False,\n",
              "                    False, False, False],\n",
              "        fill_value='?',\n",
              "             dtype=object),\n",
              " 'params': [{'min_samples_leaf': 2, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 2, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 2, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 3, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 3, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 3, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 4, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 4, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 4, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 5, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 5, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 5, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 6, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 6, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 6, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 7, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 7, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 7, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 8, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 8, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 8, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 9, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 9, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 9, 'n_estimators': 1000},\n",
              "  {'min_samples_leaf': 10, 'n_estimators': 200},\n",
              "  {'min_samples_leaf': 10, 'n_estimators': 300},\n",
              "  {'min_samples_leaf': 10, 'n_estimators': 1000}],\n",
              " 'rank_test_score': array([ 8,  7,  3,  4,  2,  1, 12,  5,  6, 11,  9, 10, 15, 14, 13, 18, 19,\n",
              "        16, 22, 20, 17, 23, 25, 21, 27, 26, 24], dtype=int32),\n",
              " 'split0_test_score': array([0.9       , 0.9       , 0.90096154, 0.9       , 0.89807692,\n",
              "        0.89903846, 0.90192308, 0.9       , 0.90096154, 0.89807692,\n",
              "        0.89807692, 0.89326923, 0.89807692, 0.90096154, 0.89711538,\n",
              "        0.90288462, 0.9       , 0.90192308, 0.89230769, 0.8875    ,\n",
              "        0.89423077, 0.89903846, 0.89519231, 0.89615385, 0.89519231,\n",
              "        0.88942308, 0.89519231]),\n",
              " 'split1_test_score': array([0.88653846, 0.89326923, 0.89423077, 0.8875    , 0.89230769,\n",
              "        0.89038462, 0.88076923, 0.88846154, 0.88653846, 0.88942308,\n",
              "        0.89230769, 0.89230769, 0.88557692, 0.8875    , 0.88846154,\n",
              "        0.88269231, 0.88942308, 0.88846154, 0.88461538, 0.88557692,\n",
              "        0.89326923, 0.88365385, 0.88557692, 0.8875    , 0.88173077,\n",
              "        0.88365385, 0.88942308]),\n",
              " 'split2_test_score': array([0.89807692, 0.89903846, 0.89615385, 0.89134615, 0.89615385,\n",
              "        0.89903846, 0.88942308, 0.89615385, 0.89519231, 0.89230769,\n",
              "        0.89230769, 0.89423077, 0.88942308, 0.89519231, 0.89230769,\n",
              "        0.88365385, 0.88269231, 0.88846154, 0.88942308, 0.8875    ,\n",
              "        0.88653846, 0.88653846, 0.88461538, 0.88653846, 0.87884615,\n",
              "        0.88173077, 0.88269231]),\n",
              " 'split3_test_score': array([0.875     , 0.87403846, 0.87980769, 0.87884615, 0.88076923,\n",
              "        0.88173077, 0.87980769, 0.88173077, 0.87788462, 0.87692308,\n",
              "        0.87307692, 0.87596154, 0.87115385, 0.87403846, 0.87211538,\n",
              "        0.86730769, 0.87019231, 0.87115385, 0.86923077, 0.87211538,\n",
              "        0.87115385, 0.86923077, 0.86826923, 0.86538462, 0.86153846,\n",
              "        0.86634615, 0.86826923]),\n",
              " 'split4_test_score': array([0.89220404, 0.89509143, 0.89701636, 0.89220404, 0.89316651,\n",
              "        0.89412897, 0.89701636, 0.89797883, 0.89509143, 0.89894129,\n",
              "        0.89990375, 0.88739172, 0.88931665, 0.88739172, 0.89027911,\n",
              "        0.88642926, 0.88739172, 0.88161694, 0.88835419, 0.89027911,\n",
              "        0.88354187, 0.88546679, 0.8825794 , 0.88546679, 0.88642926,\n",
              "        0.88450433, 0.87969201]),\n",
              " 'split5_test_score': array([0.88835419, 0.88450433, 0.88546679, 0.88931665, 0.89124158,\n",
              "        0.88546679, 0.88546679, 0.88450433, 0.88065448, 0.87680462,\n",
              "        0.87584216, 0.87584216, 0.87776708, 0.88450433, 0.87680462,\n",
              "        0.87487969, 0.87680462, 0.8719923 , 0.87872955, 0.87680462,\n",
              "        0.87487969, 0.87295476, 0.87487969, 0.87391723, 0.86717998,\n",
              "        0.87391723, 0.87487969]),\n",
              " 'split6_test_score': array([0.89412897, 0.8960539 , 0.89894129, 0.89509143, 0.89412897,\n",
              "        0.90375361, 0.89027911, 0.8960539 , 0.89797883, 0.89701636,\n",
              "        0.89509143, 0.89990375, 0.89316651, 0.89124158, 0.89701636,\n",
              "        0.8960539 , 0.88835419, 0.89412897, 0.88546679, 0.88739172,\n",
              "        0.89316651, 0.88931665, 0.89027911, 0.89509143, 0.88354187,\n",
              "        0.88931665, 0.89220404]),\n",
              " 'split7_test_score': array([0.90375361, 0.90086622, 0.90375361, 0.91530318, 0.91241578,\n",
              "        0.91145332, 0.90760346, 0.90471607, 0.90856593, 0.90471607,\n",
              "        0.90760346, 0.90760346, 0.90375361, 0.90182868, 0.906641  ,\n",
              "        0.90375361, 0.90471607, 0.90471607, 0.90086622, 0.90086622,\n",
              "        0.90471607, 0.89894129, 0.89894129, 0.89894129, 0.90375361,\n",
              "        0.90182868, 0.90086622]),\n",
              " 'split8_test_score': array([0.89412897, 0.89027911, 0.89701636, 0.89701636, 0.89797883,\n",
              "        0.89412897, 0.88450433, 0.88931665, 0.89220404, 0.88161694,\n",
              "        0.89220404, 0.89220404, 0.88546679, 0.89027911, 0.88835419,\n",
              "        0.87969201, 0.8825794 , 0.88354187, 0.8825794 , 0.88739172,\n",
              "        0.8825794 , 0.87969201, 0.88354187, 0.8825794 , 0.88161694,\n",
              "        0.87969201, 0.88065448]),\n",
              " 'split9_test_score': array([0.89027911, 0.89027911, 0.88642926, 0.8825794 , 0.88450433,\n",
              "        0.89124158, 0.88354187, 0.88835419, 0.88835419, 0.88642926,\n",
              "        0.88546679, 0.88739172, 0.88642926, 0.88161694, 0.88835419,\n",
              "        0.89124158, 0.88450433, 0.89220404, 0.88065448, 0.8825794 ,\n",
              "        0.88642926, 0.88354187, 0.88065448, 0.8825794 , 0.88354187,\n",
              "        0.88546679, 0.88161694]),\n",
              " 'std_fit_time': array([ 2.93407533,  3.65629833, 12.46858924,  0.70843528,  0.63298585,\n",
              "         1.13198026,  0.30846294,  0.38323658,  1.64481554,  0.83364994,\n",
              "         0.45169737,  1.26624119,  0.4502105 ,  0.96914246,  1.32888563,\n",
              "         0.58907839,  0.39243715,  0.77705785,  0.47017223,  0.41817215,\n",
              "         1.36280506,  0.46273677,  0.71426233,  1.21243429,  0.37698051,\n",
              "         0.54903978,  0.88804273]),\n",
              " 'std_score_time': array([0.00573619, 0.01682526, 0.05399365, 0.00805766, 0.01827037,\n",
              "        0.02262487, 0.00964891, 0.0095663 , 0.02511984, 0.00514459,\n",
              "        0.00743926, 0.04667918, 0.00563226, 0.02743909, 0.01946819,\n",
              "        0.00323467, 0.01998525, 0.06296518, 0.00774363, 0.00965106,\n",
              "        0.03349952, 0.0026954 , 0.00821029, 0.03592082, 0.00410999,\n",
              "        0.00639713, 0.01761027]),\n",
              " 'std_test_score': array([0.0076332 , 0.00776728, 0.00722807, 0.00960718, 0.00807826,\n",
              "        0.00826947, 0.00884998, 0.00696063, 0.00883282, 0.00918844,\n",
              "        0.01003944, 0.00924196, 0.00885117, 0.00808696, 0.00939838,\n",
              "        0.01118456, 0.0095936 , 0.01062675, 0.00805328, 0.00732467,\n",
              "        0.00934606, 0.00915566, 0.00856042, 0.00973152, 0.01148463,\n",
              "        0.00901884, 0.0093339 ])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ofOEvCk7FTHl",
        "outputId": "aa1d343f-d47b-483a-ddf1-a384fde737b0"
      },
      "source": [
        "rf_grid2.best_params_  # Parameter setting that gave the best results on the hold out data.\n",
        "print(rf_grid2.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'min_samples_leaf': 2, 'n_estimators': 400}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nh_L3VDt9wqJ",
        "outputId": "07598b50-a7ae-4438-da05-e8b016b285cd"
      },
      "source": [
        "rf_grid3.best_params_  # Parameter setting that gave the best results on the hold out data.\n",
        "print(rf_grid3.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'min_samples_leaf': 2, 'n_estimators': 700}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N4wr6NJ2z1H1"
      },
      "source": [
        "## Comparing the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XvQ5c0WOhGka",
        "outputId": "f87e1f63-20c0-4f83-e5e4-2e35bfdb508d"
      },
      "source": [
        "random_f1 = rf_grid1.best_estimator_ \n",
        "print(\"Random Forest 1000 estimators, min leaf 3\")\n",
        "predrf_grid1 = random_f1.predict(x_test)\n",
        "print(\"Confusion Matrix for Random Forest Classifier:\")\n",
        "print(confusion_matrix(y_test,predrf_grid1))\n",
        "print(\"Score:\",round(accuracy_score(y_test,predrf_grid1)*100,2))\n",
        "print(\"Classification Report:\")\n",
        "print(classification_report(y_test,predrf_grid1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 1000 estimators, min leaf 3\n",
            "Confusion Matrix for Random Forest Classifier:\n",
            "[[1162  114]\n",
            " [ 157 1166]]\n",
            "Score: 89.57\n",
            "Classification Report:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.91      0.90      1276\n",
            "           1       0.91      0.88      0.90      1323\n",
            "\n",
            "    accuracy                           0.90      2599\n",
            "   macro avg       0.90      0.90      0.90      2599\n",
            "weighted avg       0.90      0.90      0.90      2599\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qObDQ8TXGl4q",
        "outputId": "0573380d-f9da-46da-a59d-e7658b54409a"
      },
      "source": [
        "random_f2 = rf_grid2.best_estimator_ \n",
        "print(\"Random Forest 400 estimators, min leaf 2\")\n",
        "predrf_grid2 = random_f2.predict(x_test)\n",
        "print(\"Confusion Matrix for Random Forest Classifier:\")\n",
        "print(confusion_matrix(y_test,predrf_grid2))\n",
        "print(\"Score:\",round(accuracy_score(y_test,predrf_grid2)*100,2))\n",
        "print(\"Classification Report:\")\n",
        "print(classification_report(y_test,predrf_grid2))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 400 estimators, min leaf 2\n",
            "Confusion Matrix for Random Forest Classifier:\n",
            "[[1161  115]\n",
            " [ 158 1165]]\n",
            "Score: 89.5\n",
            "Classification Report:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.91      0.89      1276\n",
            "           1       0.91      0.88      0.90      1323\n",
            "\n",
            "    accuracy                           0.89      2599\n",
            "   macro avg       0.90      0.90      0.89      2599\n",
            "weighted avg       0.90      0.89      0.89      2599\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGBrwBCi985C",
        "outputId": "33429fb5-6d7c-4dfb-e8d2-ace22ebac63b"
      },
      "source": [
        " random_f3 = rf_grid3.best_estimator_ \n",
        "print(\"Random Forest 700 estimators, min leaf 2\")\n",
        "predrf_grid3 = random_f3.predict(x_test)\n",
        "print(\"Confusion Matrix for Random Forest Classifier:\")\n",
        "print(confusion_matrix(y_test,predrf_grid3))\n",
        "print(\"Score:\",round(accuracy_score(y_test,predrf_grid3)*100,2))\n",
        "print(\"Classification Report:\")\n",
        "print(classification_report(y_test,predrf_grid3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 700 estimators, min leaf 2\n",
            "Confusion Matrix for Random Forest Classifier:\n",
            "[[1163  113]\n",
            " [ 163 1160]]\n",
            "Score: 89.38\n",
            "Classification Report:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.91      0.89      1276\n",
            "           1       0.91      0.88      0.89      1323\n",
            "\n",
            "    accuracy                           0.89      2599\n",
            "   macro avg       0.89      0.89      0.89      2599\n",
            "weighted avg       0.89      0.89      0.89      2599\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f9uwPiezhgiY",
        "outputId": "506edbcd-3dec-4e57-b372-fd10ba2cb13c"
      },
      "source": [
        "print(\"Random Forest 1000 estimators, min leaf 3\")\n",
        "print(\"Test  Accuracy : %f\"%random_f1.score(x_test, y_test))\n",
        "print(\"Train Accuracy : %f\"%random_f1.score(x_train, y_train))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 1000 estimators, min leaf 3\n",
            "Test  Accuracy : 0.895729\n",
            "Train Accuracy : 0.951703\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0WFJB9q8Gx-l",
        "outputId": "00ea13c9-afd2-4822-9f86-73c72cae0252"
      },
      "source": [
        "print(\"Random Forest 400 estimators, min leaf 2\")\n",
        "print(\"Test  Accuracy : %f\"%random_f2.score(x_test, y_test))\n",
        "print(\"Train Accuracy : %f\"%random_f2.score(x_train, y_train))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 400 estimators, min leaf 2\n",
            "Test  Accuracy : 0.894960\n",
            "Train Accuracy : 0.976910\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hoqofTjr-HKC",
        "outputId": "20b9116c-fe99-43ee-8033-073e5085955e"
      },
      "source": [
        "print(\"Random Forest 700 estimators, min leaf 2\")\n",
        "print(\"Test  Accuracy : %f\"%random_f3.score(x_test, y_test))\n",
        "print(\"Train Accuracy : %f\"%random_f3.score(x_train, y_train))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Random Forest 700 estimators, min leaf 2\n",
            "Test  Accuracy : 0.893805\n",
            "Train Accuracy : 0.977391\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "113BH-shPm2J"
      },
      "source": [
        "## Comparing feature importances"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mf1B9UpC9beu"
      },
      "source": [
        "import pickle\n",
        "with open('/content/drive/MyDrive/Yelp/model_128_/feature_names_feb_03.pkl', 'rb') as f:\n",
        "  feature_names = pickle.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gvVdKlPREyvo"
      },
      "source": [
        "with open('/content/drive/MyDrive/Yelp/model_128_/unique_ids_feb_03.pkl', 'rb') as f:\n",
        "  unique_ids = pickle.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_a4L0aWF1Sp0"
      },
      "source": [
        "importance_rf1000 = random_f1.feature_importances_\n",
        "importance_rf400 = random_f2.feature_importances_\n",
        "importance_rf700 = random_f3.feature_importances_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kI0QZ6FUEunR"
      },
      "source": [
        "feature_importance_1000 = pd.DataFrame({'ids': unique_ids, 'words': feature_names, 'imp': importance_rf1000})\n",
        "feature_importance_400 = pd.DataFrame({'ids': unique_ids, 'words': feature_names, 'imp': importance_rf400})\n",
        "feature_importance_700 = pd.DataFrame({'ids': unique_ids, 'words': feature_names, 'imp': importance_rf700})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "id": "4vxqkaIdJlHN",
        "outputId": "d326ad8d-0b3a-439e-e96e-32fd28731585"
      },
      "source": [
        "# top 15 words\n",
        "feature_importance_1000.sort_values(by=['imp'], ascending=False)[:15] #words with important role "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ids</th>\n",
              "      <th>words</th>\n",
              "      <th>imp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>2307</td>\n",
              "      <td>great</td>\n",
              "      <td>0.042505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8020</th>\n",
              "      <td>12090</td>\n",
              "      <td>delicious</td>\n",
              "      <td>0.020511</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3802</th>\n",
              "      <td>6429</td>\n",
              "      <td>amazing</td>\n",
              "      <td>0.018595</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2978</th>\n",
              "      <td>5409</td>\n",
              "      <td>worst</td>\n",
              "      <td>0.016944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>279</th>\n",
              "      <td>2293</td>\n",
              "      <td>love</td>\n",
              "      <td>0.015892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2954</th>\n",
              "      <td>5379</td>\n",
              "      <td>friendly</td>\n",
              "      <td>0.015123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>373</th>\n",
              "      <td>2409</td>\n",
              "      <td>told</td>\n",
              "      <td>0.014258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5957</th>\n",
              "      <td>9202</td>\n",
              "      <td>horrible</td>\n",
              "      <td>0.014049</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>327</th>\n",
              "      <td>2356</td>\n",
              "      <td>asked</td>\n",
              "      <td>0.013634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3991</th>\n",
              "      <td>6659</td>\n",
              "      <td>terrible</td>\n",
              "      <td>0.013311</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8440</th>\n",
              "      <td>12726</td>\n",
              "      <td>rude</td>\n",
              "      <td>0.012900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193</th>\n",
              "      <td>2190</td>\n",
              "      <td>best</td>\n",
              "      <td>0.012858</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>2056</td>\n",
              "      <td>said</td>\n",
              "      <td>0.012458</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3305</th>\n",
              "      <td>5791</td>\n",
              "      <td>definitely</td>\n",
              "      <td>0.011669</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>682</th>\n",
              "      <td>2781</td>\n",
              "      <td>minutes</td>\n",
              "      <td>0.010459</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ids       words       imp\n",
              "288    2307       great  0.042505\n",
              "8020  12090   delicious  0.020511\n",
              "3802   6429     amazing  0.018595\n",
              "2978   5409       worst  0.016944\n",
              "279    2293        love  0.015892\n",
              "2954   5379    friendly  0.015123\n",
              "373    2409        told  0.014258\n",
              "5957   9202    horrible  0.014049\n",
              "327    2356       asked  0.013634\n",
              "3991   6659    terrible  0.013311\n",
              "8440  12726        rude  0.012900\n",
              "193    2190        best  0.012858\n",
              "82     2056        said  0.012458\n",
              "3305   5791  definitely  0.011669\n",
              "682    2781     minutes  0.010459"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "id": "6SutrbwDIGDk",
        "outputId": "1041d746-2b7c-4796-f149-b59f4f4ee78e"
      },
      "source": [
        "# top 15 words\n",
        "feature_importance_400.sort_values(by=['imp'], ascending=False)[:15] #words with important role "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ids</th>\n",
              "      <th>words</th>\n",
              "      <th>imp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>2307</td>\n",
              "      <td>great</td>\n",
              "      <td>0.035311</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8020</th>\n",
              "      <td>12090</td>\n",
              "      <td>delicious</td>\n",
              "      <td>0.018375</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3802</th>\n",
              "      <td>6429</td>\n",
              "      <td>amazing</td>\n",
              "      <td>0.016065</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2978</th>\n",
              "      <td>5409</td>\n",
              "      <td>worst</td>\n",
              "      <td>0.014470</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2954</th>\n",
              "      <td>5379</td>\n",
              "      <td>friendly</td>\n",
              "      <td>0.012964</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>279</th>\n",
              "      <td>2293</td>\n",
              "      <td>love</td>\n",
              "      <td>0.012546</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5957</th>\n",
              "      <td>9202</td>\n",
              "      <td>horrible</td>\n",
              "      <td>0.012401</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3991</th>\n",
              "      <td>6659</td>\n",
              "      <td>terrible</td>\n",
              "      <td>0.012306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>327</th>\n",
              "      <td>2356</td>\n",
              "      <td>asked</td>\n",
              "      <td>0.012191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>373</th>\n",
              "      <td>2409</td>\n",
              "      <td>told</td>\n",
              "      <td>0.011806</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8440</th>\n",
              "      <td>12726</td>\n",
              "      <td>rude</td>\n",
              "      <td>0.011279</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193</th>\n",
              "      <td>2190</td>\n",
              "      <td>best</td>\n",
              "      <td>0.011134</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3305</th>\n",
              "      <td>5791</td>\n",
              "      <td>definitely</td>\n",
              "      <td>0.010732</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>2056</td>\n",
              "      <td>said</td>\n",
              "      <td>0.010580</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3925</th>\n",
              "      <td>6581</td>\n",
              "      <td>excellent</td>\n",
              "      <td>0.009408</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ids       words       imp\n",
              "288    2307       great  0.035311\n",
              "8020  12090   delicious  0.018375\n",
              "3802   6429     amazing  0.016065\n",
              "2978   5409       worst  0.014470\n",
              "2954   5379    friendly  0.012964\n",
              "279    2293        love  0.012546\n",
              "5957   9202    horrible  0.012401\n",
              "3991   6659    terrible  0.012306\n",
              "327    2356       asked  0.012191\n",
              "373    2409        told  0.011806\n",
              "8440  12726        rude  0.011279\n",
              "193    2190        best  0.011134\n",
              "3305   5791  definitely  0.010732\n",
              "82     2056        said  0.010580\n",
              "3925   6581   excellent  0.009408"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "id": "50ibQyys_r14",
        "outputId": "3006fad6-b107-4d54-eb7d-2ec1e7969096"
      },
      "source": [
        "# top 15 words\n",
        "feature_importance_700.sort_values(by=['imp'], ascending=False)[:15] #words with important role "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ids</th>\n",
              "      <th>words</th>\n",
              "      <th>imp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>2307</td>\n",
              "      <td>great</td>\n",
              "      <td>0.036491</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8020</th>\n",
              "      <td>12090</td>\n",
              "      <td>delicious</td>\n",
              "      <td>0.017180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3802</th>\n",
              "      <td>6429</td>\n",
              "      <td>amazing</td>\n",
              "      <td>0.015990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2978</th>\n",
              "      <td>5409</td>\n",
              "      <td>worst</td>\n",
              "      <td>0.014070</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>279</th>\n",
              "      <td>2293</td>\n",
              "      <td>love</td>\n",
              "      <td>0.013364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2954</th>\n",
              "      <td>5379</td>\n",
              "      <td>friendly</td>\n",
              "      <td>0.012707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5957</th>\n",
              "      <td>9202</td>\n",
              "      <td>horrible</td>\n",
              "      <td>0.011838</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>327</th>\n",
              "      <td>2356</td>\n",
              "      <td>asked</td>\n",
              "      <td>0.011753</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3991</th>\n",
              "      <td>6659</td>\n",
              "      <td>terrible</td>\n",
              "      <td>0.011505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>373</th>\n",
              "      <td>2409</td>\n",
              "      <td>told</td>\n",
              "      <td>0.011433</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193</th>\n",
              "      <td>2190</td>\n",
              "      <td>best</td>\n",
              "      <td>0.011208</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8440</th>\n",
              "      <td>12726</td>\n",
              "      <td>rude</td>\n",
              "      <td>0.011158</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>2056</td>\n",
              "      <td>said</td>\n",
              "      <td>0.010726</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3305</th>\n",
              "      <td>5791</td>\n",
              "      <td>definitely</td>\n",
              "      <td>0.010477</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>682</th>\n",
              "      <td>2781</td>\n",
              "      <td>minutes</td>\n",
              "      <td>0.009058</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ids       words       imp\n",
              "288    2307       great  0.036491\n",
              "8020  12090   delicious  0.017180\n",
              "3802   6429     amazing  0.015990\n",
              "2978   5409       worst  0.014070\n",
              "279    2293        love  0.013364\n",
              "2954   5379    friendly  0.012707\n",
              "5957   9202    horrible  0.011838\n",
              "327    2356       asked  0.011753\n",
              "3991   6659    terrible  0.011505\n",
              "373    2409        told  0.011433\n",
              "193    2190        best  0.011208\n",
              "8440  12726        rude  0.011158\n",
              "82     2056        said  0.010726\n",
              "3305   5791  definitely  0.010477\n",
              "682    2781     minutes  0.009058"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qnfw0yoZ7tDv"
      },
      "source": [
        "## Saving contributions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "geU89LF28ogN"
      },
      "source": [
        "# saving the features data frame for the best random forest\n",
        "ft_imp.to_pickle('/content/drive/MyDrive/Yelp/random_forest/feature_importance_1000.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iMAxptE_C3YT",
        "outputId": "afb37424-12dc-4149-de1d-f7ec1498d2d3"
      },
      "source": [
        "%%time\n",
        "preds, bias, contributions = ti.predict(random_f1, x_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 49min 5s, sys: 1min 36s, total: 50min 42s\n",
            "Wall time: 50min 53s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uRnxuSrPDA2m"
      },
      "source": [
        "with open('/content/drive/MyDrive/Yelp/random_forest/pred_tr.pkl', 'wb') as f:\n",
        "  pickle.dump(preds, f)\n",
        "with open('/content/drive/MyDrive/Yelp/random_forest/bias_tr.pkl', 'wb') as f:\n",
        "  pickle.dump(bias, f)\n",
        "with open('/content/drive/MyDrive/Yelp/random_forest/contribs_tr.pkl', 'wb') as f:\n",
        "  pickle.dump(contributions, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OfXPm0ce73AQ"
      },
      "source": [
        "%%time\n",
        "preds_ts, bias_ts, contributions_ts = ti.predict(random_f1, x_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90g1n15C8CFx"
      },
      "source": [
        "import pickle\n",
        "with open('/content/drive/MyDrive/Yelp/random_forest/pred_ts.pkl', 'wb') as f:\n",
        "  pickle.dump(preds_ts, f)\n",
        "with open('/content/drive/MyDrive/Yelp/random_forest/bias_ts.pkl', 'wb') as f:\n",
        "  pickle.dump(bias_ts, f)\n",
        "  with open('/content/drive/MyDrive/Yelp/random_forest/contribs_ts.pkl', 'wb') as f:\n",
        "  pickle.dump(contributions_ts, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2g9d6WF16zvn",
        "outputId": "ea682dbb-0dbd-46f8-8cbd-0045ef4a3cfe"
      },
      "source": [
        "contributions[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.00046256,  0.00046256],\n",
              "       [ 0.        ,  0.        ],\n",
              "       [ 0.        ,  0.        ],\n",
              "       ...,\n",
              "       [ 0.        ,  0.        ],\n",
              "       [ 0.        ,  0.        ],\n",
              "       [ 0.        ,  0.        ]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eg4P1yVi7Cc-",
        "outputId": "5475dad7-b72a-453c-ed8f-7c5834e65ff2"
      },
      "source": [
        "type(contributions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "numpy.ndarray"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    }
  ]
}