Homework4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/smart_open/ssh.py:34: UserWarning: paramiko missing, opening SSH/SCP/SFTP paths will be disabled.  `pip install paramiko` to suppress\n",
      "  warnings.warn('paramiko missing, opening SSH/SCP/SFTP paths will be disabled.  `pip install paramiko` to suppress')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer,ENGLISH_STOP_WORDS\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix, classification_report,accuracy_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from scipy.sparse import coo_matrix, hstack\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer\n",
    "import re\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import xgboost\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer\n",
    "import nltk\n",
    "words = set(nltk.corpus.words.words())\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the datasets\n",
    "df = pd.read_csv(\"./Data/reddit_200k_train.csv\",encoding='iso-8859-1')\n",
    "test = pd.read_csv(\"./Data/reddit_200k_test.csv\",encoding='iso-8859-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Strip unnecessary columns\n",
    "df = df.drop([df.columns[i] for i in range(len(df.columns)) if i != 1 and i != 7],axis = 1)\n",
    "test = test.drop([test.columns[i] for i in range(len(test.columns)) if i != 1 and i != 7],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate the predictive feature and the targeted feature\n",
    "X_list = df[\"body\"].tolist()\n",
    "X = df[\"body\"]\n",
    "y = df[\"REMOVED\"]\n",
    "X_n = df['body'].str.replace('\\d+','')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 1 Bag of Words and simple Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1: Baseline Model (Logistic Regression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenization\n",
    "vect = CountVectorizer()\n",
    "cv_X = vect.fit_transform(X)\n",
    "\n",
    "# Split the dataset into the training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(cv_X,y, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6886087434042452"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build a baseline model (logistic regression)\n",
    "lg = LogisticRegression()\n",
    "\n",
    "# Fit the model to the data\n",
    "lg.fit(X_train, y_train)\n",
    "\n",
    "# Compute the accurecy \n",
    "lg.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6886087434042452"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute the test score\n",
    "y_pred = lg.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[19777,  5859],\n",
       "       [ 7183,  9064]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build a confusion matrix\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False       0.73      0.77      0.75     25636\n",
      "        True       0.61      0.56      0.58     16247\n",
      "\n",
      "   micro avg       0.69      0.69      0.69     41883\n",
      "   macro avg       0.67      0.66      0.67     41883\n",
      "weighted avg       0.68      0.69      0.69     41883\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The table of each test score\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2: Rescale and Use Other Techniques (n-grams, tf-idf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Strip punctuations and stopwords without n_gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a countvectorizer with arguments of removing punctuations and stopwords\n",
    "vect = CountVectorizer(token_pattern=r\"\\b\\w+\\b\", stop_words=ENGLISH_STOP_WORDS, min_df=4)\n",
    "X_n_cv = vect.fit_transform(X_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6833082634959291"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Split the dataset into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_cv,y, random_state = 0)\n",
    "\n",
    "# Build a logistic model and fit it to the data\n",
    "lg = LogisticRegression()\n",
    "lg.fit(X_train, y_train)\n",
    "\n",
    "# Compute the test score\n",
    "lg.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.5s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.6s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.5s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    1.4s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.6s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    2.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.543320952797687, total=   0.7s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    3.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.545242171679164, total=   0.9s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5436764386427085, total=   0.9s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    5.4s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7141468589305742, total=   0.9s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    6.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7214084651395443, total=   0.9s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    7.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7169638571093843, total=   0.8s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    8.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7571239869393849, total=   1.1s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7588402902349063, total=   1.0s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7554309139610973, total=   1.5s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7232123744950699, total=   2.4s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7216165450123482, total=   1.5s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7200405552795927, total=   3.1s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.68735210005142, total=   8.2s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6861621522619682, total=   7.2s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6841141845653332, total=   8.6s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.675826573453972, total=  21.7s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6748955641104528, total=  12.6s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6727147857190562, total=  17.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:  1.6min finished\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('lg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'lg__penalty': ['l1'], 'lg__C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='roc_auc', verbose=10)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Gridsearch \n",
    "pipe = Pipeline([\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"lg\", LogisticRegression())])\n",
    "\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"lg__penalty\":[\"l1\"],\n",
    "    \"lg__C\":np.logspace(-3,3,7)\n",
    "\n",
    "}\n",
    "\n",
    "\n",
    "grid = GridSearchCV(pipe, param_grid=param_grid, cv=3 , scoring=\"roc_auc\", verbose=10)\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_cv,y, random_state = 0)\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lg__C': 1.0, 'lg__penalty': 'l1'}\n",
      "Accurancy: 0.6949836449155982\n"
     ]
    }
   ],
   "source": [
    "# The best parameters and accuracy score \n",
    "final_model = grid.best_estimator_\n",
    "y_pred = grid.predict(X_test)\n",
    "print(grid.best_params_ )\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Strip punctuations and stopwords with bi_gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a countVectorizer with arguments of removing punctuations and stopwords and bi-gram\n",
    "vect = CountVectorizer(token_pattern=r\"\\b\\w+\\b\", stop_words=ENGLISH_STOP_WORDS, min_df=4, ngram_range=(1,2))\n",
    "X_n_cv = vect.fit_transform(X_n)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6847885777045579"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Split the dataset into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_cv,y, random_state = 0)\n",
    "\n",
    "# Build a logistic model and fit it to the data\n",
    "lg = LogisticRegression()\n",
    "lg.fit(X_train, y_train)\n",
    "\n",
    "# Compute the test score\n",
    "print(\"Accuracy: {}\".format(lg.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   1.0s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.3s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   1.0s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    2.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   1.0s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    3.8s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5200629884521171, total=   1.1s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    5.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5198304689322654, total=   1.4s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    6.8s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5192083365736061, total=   1.5s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    8.6s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7049510086449231, total=   1.5s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   10.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.712579769644371, total=   2.3s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   13.0s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7086567501579834, total=   1.9s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   15.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7571157638944396, total=   3.0s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7595311984040095, total=   1.9s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.756901679814624, total=   1.7s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.711344983553862, total=   3.9s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7115647019000113, total=   4.9s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7076470468629725, total=   6.1s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6762409862943093, total=  10.8s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6768835219292323, total=   9.0s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6703638537038803, total=  10.8s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6613833368361557, total=  17.1s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6618883625646825, total=  15.3s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.655401593091671, total=  15.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:  2.0min finished\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('lg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'lg__penalty': ['l1'], 'lg__C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='roc_auc', verbose=10)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Gridsearch\n",
    "pipe = Pipeline([\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"lg\", LogisticRegression())])\n",
    "\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "#     \"cv__token_pattern\":[r\"\\b\\w+\\b\"],\n",
    "#     \"cv__min_df\":np.arange(4,8,2),\n",
    "#     \"cv__stop_words\":[ENGLISH_STOP_WORDS]\n",
    "    \"lg__penalty\":[\"l1\"],\n",
    "    \"lg__C\":np.logspace(-3,3,7)\n",
    "\n",
    "}\n",
    "grid = GridSearchCV(pipe, param_grid=param_grid, cv=3 , scoring=\"roc_auc\", verbose=10)\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_cv,y, random_state = 0)\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lg__C': 1.0, 'lg__penalty': 'l1'}\n",
      "Accurancy: 0.69593868634052\n"
     ]
    }
   ],
   "source": [
    "# The best parameters and accuracy score \n",
    "final_model = grid.best_estimator_\n",
    "y_pred = grid.predict(X_test)\n",
    "print(grid.best_params_ )\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.3: Adding New Features (URL and Numbers) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function that checks if a string contains an url\n",
    "def findurl(string): \n",
    "    url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+] |[!*\\(\\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', string) \n",
    "    return url \n",
    "\n",
    "# A function that checks if a number(s) are  in a string\n",
    "def findnum(string):\n",
    "    num = [str(i) for i in range(10) if str(i) in string]\n",
    "    return num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Adding extra features\n",
    "df['url'] = -1\n",
    "df[\"num\"] = -1\n",
    "for i in range(df.shape[0]):\n",
    "    st = df.iloc[i,0]\n",
    "    result = findurl(st)\n",
    "    result1 = findnum(st)\n",
    "    if result != []:\n",
    "        df.iloc[i,-2] = 1\n",
    "    else:\n",
    "        df.iloc[i,-2] = 0\n",
    "        \n",
    "    if result1 != []:\n",
    "        df.iloc[i,-1] = 1\n",
    "    else:\n",
    "        df.iloc[i,-1] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "num = df[\"num\"]\n",
    "url = df[\"url\"]\n",
    "\n",
    "r = [[num[i],url[i]] for i in range(len(num))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(token_pattern=r\"\\b\\w+\\b\", stop_words=ENGLISH_STOP_WORDS, min_df=4)\n",
    "X_n_cv = vect.fit_transform(X_n)\n",
    "\n",
    "X_n_cv = hstack([X_n_cv,r])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.4s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.3s\n",
      "[CV] lg__C=0.001, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    1.0s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ........... lg__C=0.001, lg__penalty=l1, score=0.5, total=   0.4s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    1.4s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5531012670985127, total=   0.4s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    1.9s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.5520064177220975, total=   0.5s\n",
      "[CV] lg__C=0.01, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    2.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.01, lg__penalty=l1, score=0.55245930848009, total=   0.5s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    3.1s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.718794809410702, total=   0.6s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    3.8s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7109608045513142, total=   0.6s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    4.6s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.7195023952145924, total=   0.6s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    5.3s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7573855941397839, total=   1.6s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7528435090333897, total=   1.5s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.7578018654843406, total=   1.2s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7226647315623086, total=   2.1s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7179227553408662, total=   2.6s\n",
      "[CV] lg__C=10.0, lg__penalty=l1 ......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=10.0, lg__penalty=l1, score=0.7237819191192159, total=   2.5s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6835210004996994, total=   3.9s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6830174899568582, total=   4.5s\n",
      "[CV] lg__C=100.0, lg__penalty=l1 .....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=100.0, lg__penalty=l1, score=0.6840735473302715, total=   4.9s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6719418174848119, total=   7.2s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6719357116277758, total=   8.0s\n",
      "[CV] lg__C=1000.0, lg__penalty=l1 ....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1000.0, lg__penalty=l1, score=0.6711785331687943, total=   5.8s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:   52.9s finished\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('lg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'lg__penalty': ['l1'], 'lg__C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='roc_auc', verbose=10)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"lg\", LogisticRegression())])\n",
    "\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "#     \"cv__token_pattern\":[r\"\\b\\w+\\b\"],\n",
    "#     \"cv__min_df\":np.arange(4,8,2),\n",
    "#     \"cv__stop_words\":[ENGLISH_STOP_WORDS]\n",
    "    \"lg__penalty\":[\"l1\"],\n",
    "    \"lg__C\":np.logspace(-3,3,7)\n",
    "\n",
    "}\n",
    "grid = GridSearchCV(pipe, param_grid=param_grid, cv=3 , scoring=\"roc_auc\", verbose=10)\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_cv,y, random_state = 42)\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lg__C': 1.0, 'lg__penalty': 'l1'}\n",
      "Accurancy: 0.6952462813074517\n"
     ]
    }
   ],
   "source": [
    "# The best parameters and accuracy score \n",
    "final_model = grid.best_estimator_\n",
    "y_pred = grid.predict(X_test)\n",
    "print(grid.best_params_ )\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove non-English words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = set(nltk.corpus.words.words())\n",
    "X_n_n = [] #string without numbers and non english words\n",
    "for sentence in X_n:\n",
    "    sent = sentence\n",
    "    \n",
    "\n",
    "    a = \" \".join(w for w in nltk.wordpunct_tokenize(sent) \\\n",
    "            if w.lower() in words or not w.isalpha())\n",
    "    X_n_n.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(token_pattern=r\"\\b\\w+\\b\", stop_words=ENGLISH_STOP_WORDS, min_df=4)\n",
    "X_n_n_cv = vect.fit_transform(X_n_n)\n",
    "\n",
    "X_n_n_cv = hstack([X_n_n_cv,r])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "[CV] lg__penalty=l1 ..................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ......... lg__penalty=l1, score=0.7300862231110424, total=   0.5s\n",
      "[CV] lg__penalty=l1 ..................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ......... lg__penalty=l1, score=0.7271632904992076, total=   0.5s\n",
      "[CV] lg__penalty=l1 ..................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    1.1s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ......... lg__penalty=l1, score=0.7221026183479625, total=   0.5s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    1.7s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    1.7s finished\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('lg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'lg__penalty': ['l1']}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='roc_auc',\n",
       "       verbose=10)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"lg\", LogisticRegression())])\n",
    "\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"lg__penalty\":[\"l1\"],\n",
    "}\n",
    "grid = GridSearchCV(pipe, param_grid=param_grid, cv=3 , scoring=\"roc_auc\", verbose = 10)\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_n_n_cv, y, stratify = y, random_state = 0)\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lg__penalty': 'l1'}\n",
      "Accurancy: 0.6716806341475061\n"
     ]
    }
   ],
   "source": [
    "# The best parameters and accuracy score \n",
    "final_model = grid.best_estimator_\n",
    "y_pred = final_model.predict(X_test)\n",
    "print(grid.best_params_)\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Undersampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(129476, 1)\n",
      "[64738 64738]\n"
     ]
    }
   ],
   "source": [
    "rus = RandomUnderSampler(replacement=False)\n",
    "X_train_subsample, y_train_subsample = rus.fit_sample(\n",
    "    X.to_frame(), y)\n",
    "\n",
    "print(X_train_subsample.shape)\n",
    "print(np.bincount(y_train_subsample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = []\n",
    "num = []\n",
    "for i in range(X_train_subsample.shape[0]):\n",
    "    st = X_train_subsample[i][0]\n",
    "   #print(\"st\",st)\n",
    "    result = findurl(st)\n",
    "    result1 = findnum(st)\n",
    "    if result != []:\n",
    "        url.append(1)\n",
    "    else:\n",
    "        url.append(0)\n",
    "        \n",
    "    if result1 != []:\n",
    "        num.append(1)\n",
    "    else:\n",
    "        num.append(0)\n",
    "\n",
    "r = [[num[i],url[i]] for i in range(len(num))]\n",
    "\n",
    "\n",
    "X_train_subsample = pd.Series(X_train_subsample.flatten()).str.replace('\\d+','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(token_pattern=r\"\\b\\w+\\b\", stop_words=ENGLISH_STOP_WORDS, min_df=4, ngram_range=(1,3))\n",
    "X_SS_cv = vect.fit_transform(X_train_subsample)\n",
    "X_SS_cv = hstack([X_SS_cv,r])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "undersample_pipe = Pipeline([\n",
    "#     (\"cv\", CountVectorizer(token_pattern= r\"\\b\\w+\\b\",ngram_range=(1,3))),\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"lg\", LogisticRegression())])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_SS_cv, y_train_subsample, stratify = y_train_subsample,shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accurancy: 0.7003614569495505\n"
     ]
    }
   ],
   "source": [
    "undersample_pipe.fit(X_train, y_train)\n",
    "y_pred = undersample_pipe.predict(X_test)\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "[CV] lg__C=0.001 .....................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............ lg__C=0.001, score=0.6774482545566883, total=   0.5s\n",
      "[CV] lg__C=0.001 .....................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............ lg__C=0.001, score=0.6732676326114492, total=   0.4s\n",
      "[CV] lg__C=0.001 .....................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    1.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............ lg__C=0.001, score=0.6777063766683143, total=   0.4s\n",
      "[CV] lg__C=0.01 ......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    1.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. lg__C=0.01, score=0.6810318195860364, total=   0.4s\n",
      "[CV] lg__C=0.01 ......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    2.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. lg__C=0.01, score=0.6766659458123513, total=   0.4s\n",
      "[CV] lg__C=0.01 ......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    2.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. lg__C=0.01, score=0.6795600593178448, total=   0.4s\n",
      "[CV] lg__C=0.1 .......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    3.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. lg__C=0.1, score=0.6905468025949953, total=   0.6s\n",
      "[CV] lg__C=0.1 .......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    3.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. lg__C=0.1, score=0.6861812227748771, total=   0.6s\n",
      "[CV] lg__C=0.1 .......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    4.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. lg__C=0.1, score=0.6924740484429066, total=   0.6s\n",
      "[CV] lg__C=1.0 .......................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    5.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. lg__C=1.0, score=0.6941921532282979, total=   0.9s\n",
      "[CV] lg__C=1.0 .......................................................\n",
      "[CV] .............. lg__C=1.0, score=0.6894250671939202, total=   0.9s\n",
      "[CV] lg__C=1.0 .......................................................\n",
      "[CV] .............. lg__C=1.0, score=0.6951309935739002, total=   1.0s\n",
      "[CV] lg__C=10.0 ......................................................\n",
      "[CV] ............. lg__C=10.0, score=0.6685511275872722, total=   1.9s\n",
      "[CV] lg__C=10.0 ......................................................\n",
      "[CV] ............. lg__C=10.0, score=0.6667799437733634, total=   1.9s\n",
      "[CV] lg__C=10.0 ......................................................\n",
      "[CV] ............. lg__C=10.0, score=0.6725469599604548, total=   2.0s\n",
      "[CV] lg__C=100.0 .....................................................\n",
      "[CV] ............ lg__C=100.0, score=0.6407784986098239, total=   4.1s\n",
      "[CV] lg__C=100.0 .....................................................\n",
      "[CV] ............ lg__C=100.0, score=0.6375544502456053, total=   4.0s\n",
      "[CV] lg__C=100.0 .....................................................\n",
      "[CV] ............... lg__C=100.0, score=0.6432896688087, total=   5.5s\n",
      "[CV] lg__C=1000.0 ....................................................\n",
      "[CV] ........... lg__C=1000.0, score=0.6251158480074143, total=   9.6s\n",
      "[CV] lg__C=1000.0 ....................................................\n",
      "[CV] ........... lg__C=1000.0, score=0.6227254471871235, total=   9.6s\n",
      "[CV] lg__C=1000.0 ....................................................\n",
      "[CV] ............ lg__C=1000.0, score=0.625370736529906, total=   8.8s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:   56.8s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('lg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'lg__C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=10)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {\n",
    "    #\"cv__max_features\":np.arange(2200,2500,100),\n",
    "#    \"cv__token_pattern\":[r\"\\b\\w+\\b\"],\n",
    "#    \"cv__min_df\":np.arange(2,8,2),\n",
    "#    \"cv__stop_words\":[ENGLISH_STOP_WORDS],\n",
    "#    \"cv__ngram_range\": [(1,3)]\n",
    "    \"lg__C\":np.logspace(-3,3,7)\n",
    "}\n",
    "grid = GridSearchCV(undersample_pipe, param_grid=param_grid, cv=3 , scoring=\"accuracy\", verbose=10)\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lg__C': 1.0}\n",
      "Accurancy: 0.7003614569495505\n"
     ]
    }
   ],
   "source": [
    "# The best parameters and accuracy score \n",
    "final_model = grid.best_estimator_\n",
    "y_pred = final_model.predict(X_test)\n",
    "print(grid.best_params_ )\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different Models "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('rf', RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=10, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_sp...obs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False))])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "#     (\"cv\", CountVectorizer(max_features=2000,min_df=5,stop_words=ENGLISH_STOP_WORDS,token_pattern=r\"\\b\\w+\\b\",ngram_range=(1,3))),\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"rf\", RandomForestClassifier(max_depth= 10, n_estimators=1000))])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"rf__n_estimators\":np.arange(1000,1500,100),\n",
    "    \"rf__max_depth\":np.arange(10,20,5),\n",
    "    \"rf__max_features\": np.arange(1000,1500,100)\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_SS_cv, y_train_subsample, stratify = y_train_subsample, random_state = 0)\n",
    "\n",
    "pipe.fit(X_train, y_train)\n",
    "\n",
    "# grid = GridSearchCV(pipe, param_grid=param_grid, cv=3, scoring=\"roc_auc\", verbose = 10)\n",
    "# grid.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6369674688745405"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = pipe.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)), ('xgb', XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1))])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "    #(\"cv\", CountVectorizer(min_df=5,ngram_range=(1,2),stop_words=ENGLISH_STOP_WORDS,token_pattern=r\"\\b\\w+\\b\")),\n",
    "    (\"tf\",TfidfTransformer()),\n",
    "    (\"xgb\", XGBClassifier())])\n",
    "\n",
    "# params = {\n",
    "#         'xgb__max_depth': np.arange(4,10,2),\n",
    "#         'xgb__eta': [0.1,0.15,0.2],\n",
    "#         'xgb__subsample': [0.5,1]\n",
    "#         }\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_SS_cv, y_train_subsample, stratify = y_train_subsample, random_state = 42)\n",
    "\n",
    "pipe.fit(X_train, y_train)\n",
    "# grid = GridSearchCV(pipe, param_grid=params, cv=3)\n",
    "# grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6404893571009299"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = pipe.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 2 Word Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_n is body data without numbers\n",
    "WE = X_n.tolist()\n",
    "sw = set(nltk.corpus.stopwords.words(\"english\"))\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "stemmer = nltk.stem.PorterStemmer() \n",
    "\n",
    "#preprocessing words lemmization and stemming\n",
    "#returing X_we, which is the stemmed and lemmazied data\n",
    "X_we = []\n",
    "count = 0\n",
    "for sentence in WE:\n",
    "#     if count%100 == 0:\n",
    "#         print(count)\n",
    "    X_we.append([])\n",
    "    s = nltk.word_tokenize(sentence)\n",
    "    for word in s:\n",
    "        if word.lower() not in sw and word.lower() not in string.punctuation:\n",
    "            current = word.lower()\n",
    "            current = lemmatizer.lemmatize(current)\n",
    "            current = stemmer.stem(current)\n",
    "            X_we[-1].append(current)\n",
    "    #count += 1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing the Word2Vec model\n",
    "wmodel = Word2Vec(sentences=X_we,\n",
    "                  sg=1,\n",
    "                  hs=0,\n",
    "                  workers=4,\n",
    "                  size=200,\n",
    "                  min_count=3,\n",
    "                  window=6,\n",
    "                  sample=1e-3,\n",
    "                  negative=5\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "#Coverting each sentence into vector points for classifier\n",
    "all_words = set(wmodel.wv.vocab.keys())\n",
    "X_we_vec = []\n",
    "for words in WE:\n",
    "    X_we_vec.append([])\n",
    "    vec = np.zeros(wmodel.vector_size)\n",
    "    nwords = 0\n",
    "    for word in words:\n",
    "        if word in all_words: \n",
    "            vec = np.add(vec, wmodel[word])\n",
    "        nwords += 1\n",
    "    vec = np.divide(vec, nwords)\n",
    "    X_we_vec[-1].append(vec.flatten())\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reformatting into dataframe\n",
    "temp = []\n",
    "for s in X_we_vec:\n",
    "    temp.append([])\n",
    "    for w in s[0]:\n",
    "        temp[-1].append(w)\n",
    "X_we_vec_df = pd.DataFrame(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#splitting data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_we_vec_df,y, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6368216221378602"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_we = LogisticRegression()\n",
    "lg_we.fit(X_train, y_train)\n",
    "lg_we.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.6161397177819223, total=  13.8s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   13.9s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.6175085388282047, total=  16.1s\n",
      "[CV] lg__C=0.1, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   30.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l1, score=0.6172458646402055, total=  14.3s\n",
      "[CV] lg__C=0.1, lg__penalty=l2 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   44.5s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l2, score=0.6161420799378785, total=   2.4s\n",
      "[CV] lg__C=0.1, lg__penalty=l2 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   47.0s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l2, score=0.6178731476874055, total=   2.6s\n",
      "[CV] lg__C=0.1, lg__penalty=l2 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   49.6s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=0.1, lg__penalty=l2, score=0.6171897613922924, total=   2.5s\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   52.2s remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  lg__C=1.0, lg__penalty=l1, score=0.665838704544025, total= 2.2min\n",
      "[CV] lg__C=1.0, lg__penalty=l1 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:  3.1min remaining:    0.0s\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "    (\"lg\", LogisticRegression())\n",
    "])\n",
    "\n",
    "param_grid = {\n",
    "\t\"lg__C\":np.logspace(-1,1,3),\n",
    "\t\"lg__penalty\":[\"l1\",\"l2\"]\n",
    "}\n",
    "\n",
    "\n",
    "grid = GridSearchCV(pipe, param_grid=param_grid, scoring='roc_auc', cv=3, verbose=10)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_we_vec_df,y, random_state = 0)\n",
    "\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "final_model = best.best_estimator_\n",
    "y_pred = final_model.predict(X_test)\n",
    "print(best.best_params_)\n",
    "print(\"Accurancy: {}\".format(accuracy_score(y_test, y_pred)))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}