TFIDF Avg W2V with 200-D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading TFIDF Avg w2v vectors(200-D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 201)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>191</th>\n",
       "      <th>192</th>\n",
       "      <th>193</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "      <th>200</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.000540</td>\n",
       "      <td>0.000332</td>\n",
       "      <td>0.000313</td>\n",
       "      <td>-0.000194</td>\n",
       "      <td>-0.000083</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>0.000859</td>\n",
       "      <td>0.000307</td>\n",
       "      <td>0.000461</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000690</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.000306</td>\n",
       "      <td>0.000165</td>\n",
       "      <td>0.000459</td>\n",
       "      <td>-0.001187</td>\n",
       "      <td>-0.000208</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.000812</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.000146</td>\n",
       "      <td>-0.000066</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-0.000457</td>\n",
       "      <td>0.000192</td>\n",
       "      <td>-0.000005</td>\n",
       "      <td>-0.000039</td>\n",
       "      <td>0.000225</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>-0.000238</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000019</td>\n",
       "      <td>0.000137</td>\n",
       "      <td>-0.000356</td>\n",
       "      <td>-0.000212</td>\n",
       "      <td>0.000220</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>-0.000381</td>\n",
       "      <td>0.000054</td>\n",
       "      <td>0.000434</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000432</td>\n",
       "      <td>-0.000329</td>\n",
       "      <td>0.000599</td>\n",
       "      <td>-0.000644</td>\n",
       "      <td>-0.000191</td>\n",
       "      <td>-0.000523</td>\n",
       "      <td>-0.000381</td>\n",
       "      <td>0.000843</td>\n",
       "      <td>-0.000436</td>\n",
       "      <td>-0.000367</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000331</td>\n",
       "      <td>-0.000169</td>\n",
       "      <td>-0.000617</td>\n",
       "      <td>0.000503</td>\n",
       "      <td>-0.000637</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>-0.000775</td>\n",
       "      <td>-0.000453</td>\n",
       "      <td>-0.000177</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000268</td>\n",
       "      <td>-0.000137</td>\n",
       "      <td>-0.000365</td>\n",
       "      <td>-0.000223</td>\n",
       "      <td>-0.000257</td>\n",
       "      <td>-0.000072</td>\n",
       "      <td>-0.000135</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>0.000237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.000024</td>\n",
       "      <td>0.000655</td>\n",
       "      <td>-0.000122</td>\n",
       "      <td>-0.000064</td>\n",
       "      <td>-0.000310</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.000723</td>\n",
       "      <td>0.000649</td>\n",
       "      <td>-0.000167</td>\n",
       "      <td>0.000996</td>\n",
       "      <td>-0.000168</td>\n",
       "      <td>0.000339</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>-0.000769</td>\n",
       "      <td>0.000997</td>\n",
       "      <td>-0.000348</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000674</td>\n",
       "      <td>-0.000085</td>\n",
       "      <td>-0.000134</td>\n",
       "      <td>0.000342</td>\n",
       "      <td>-0.000405</td>\n",
       "      <td>-0.000941</td>\n",
       "      <td>-0.000654</td>\n",
       "      <td>0.000165</td>\n",
       "      <td>0.000779</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 201 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0 -0.000540  0.000332  0.000313 -0.000194 -0.000083  0.000283 -0.000020   \n",
       "1 -0.000146 -0.000066 -0.000010 -0.000457  0.000192 -0.000005 -0.000039   \n",
       "2  0.000432 -0.000329  0.000599 -0.000644 -0.000191 -0.000523 -0.000381   \n",
       "3  0.000268 -0.000137 -0.000365 -0.000223 -0.000257 -0.000072 -0.000135   \n",
       "4 -0.000723  0.000649 -0.000167  0.000996 -0.000168  0.000339  0.000080   \n",
       "\n",
       "        7         8         9   ...        191       192       193       194  \\\n",
       "0  0.000859  0.000307  0.000461 ...  -0.000690  0.000095  0.000306  0.000165   \n",
       "1  0.000225  0.000138 -0.000238 ...  -0.000019  0.000137 -0.000356 -0.000212   \n",
       "2  0.000843 -0.000436 -0.000367 ...  -0.000331 -0.000169 -0.000617  0.000503   \n",
       "3  0.000099  0.000030  0.000237 ...   0.000011  0.000123  0.000118  0.000035   \n",
       "4 -0.000769  0.000997 -0.000348 ...  -0.000674 -0.000085 -0.000134  0.000342   \n",
       "\n",
       "        195       196       197       198       199  200  \n",
       "0  0.000459 -0.001187 -0.000208  0.000765  0.000812    1  \n",
       "1  0.000220  0.000008 -0.000381  0.000054  0.000434    1  \n",
       "2 -0.000637  0.000036 -0.000775 -0.000453 -0.000177    0  \n",
       "3 -0.000024  0.000655 -0.000122 -0.000064 -0.000310    1  \n",
       "4 -0.000405 -0.000941 -0.000654  0.000165  0.000779    1  \n",
       "\n",
       "[5 rows x 201 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_avg_w2v =pd.read_pickle(\"tfidf_avg_vec_200\")\n",
    "print(tfidf_avg_w2v.shape)\n",
    "tfidf_avg_w2v.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 200)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(122109,)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x =tfidf_avg_w2v.iloc[:,:200]\n",
    "y =tfidf_avg_w2v.iloc[:,200]\n",
    "print(x.shape)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Column Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a =StandardScaler()\n",
    "x =a.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test =train_test_split(x,y,test_size =0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(85476, 200)\n",
      "(36633, 200)\n",
      "(85476,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(36633,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tuned_params = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GridSearchCV(LogisticRegression(), tuned_params, scoring = 'accuracy')\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=0.01, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "0.696339366145\n"
     ]
    }
   ],
   "source": [
    "print(model.best_estimator_)\n",
    "print(model.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.best_estimator_.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69.633936614527883"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_pred, normalize=True) * float(100)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see increasing dimenions the model performs better than 100-D as the chances of data becomes linearly separable in higher dimensions increases.So we get the best hyperplane as dimensions increases due to which model behaves better."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11064,  5139],\n",
       "       [ 5985, 14445]], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1 regularization and sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_    \n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means 199 out of 200 are non zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "198\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.001, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.00025, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At c=0.00025 we are getting top 1 features and rest becomes zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}