truncated_svd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "# pd.options.display.max_columns = 10 \n",
    "# pd.options.display.width = 134\n",
    "# pd.options.display.max_rows = 20\n",
    "\n",
    "# import warnings\n",
    "# warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_folder = './dataset/'\n",
    "\n",
    "\n",
    "genome_scores_df=pd.read_csv(data_folder+'genome_scores.csv') # Large (500MB)\n",
    "movies_df=pd.read_csv(data_folder+'movies.csv')\n",
    "validation = pd.read_csv(data_folder+'validation.csv') \n",
    "\n",
    "df_train=pd.read_csv(data_folder+'train_with_valid.csv') # Large 500MB\n",
    "test=pd.read_csv(data_folder+'test.csv') # Large 500MB\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vishwesh/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "# matrix = pd.concat([train,test]).drop_duplicates(subset = ['userId','movieId'],keep = 'first')\n",
    "# matrix = matrix.pivot('userId','movieId','rating')\n",
    "\n",
    "# movie_means = matrix.mean()\n",
    "# user_means = matrix.mean(axis=1)\n",
    "# #Mean shifting\n",
    "# mzm = matrix-movie_means\n",
    "# mz = mzm.fillna(0)\n",
    "# #To get locations where we have ratings\n",
    "# mask = -mzm.isnull()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-c2862e32f565>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#Due to adding of validation, we have some duplicates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmovie_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop_duplicates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'userId'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'movieId'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mkeep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'first'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m#Creates a movie matrix of #numofusers vs #noofmovies\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmovie_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmovie_matrix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpivot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'userId'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'movieId'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rating'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df_train' is not defined"
     ]
    }
   ],
   "source": [
    "#Due to adding of validation, we have some duplicates\n",
    "movie_matrix = pd.concat([df_train,test]).drop_duplicates(subset = ['userId','movieId'],keep = 'first')\n",
    "#Creates a movie matrix of #numofusers vs #noofmovies\n",
    "movie_matrix = movie_matrix.pivot('userId','movieId','rating')\n",
    "\n",
    "movie_means = movie_matrix.mean()\n",
    "user_means = movie_matrix.mean(axis=1)\n",
    "#Mean shifting\n",
    "movie_shifted_temp = movie_matrix-movie_means\n",
    "movie_shifted = movie_shifted_temp.fillna(0)\n",
    "#To get locations where we have ratings\n",
    "mask = -movie_shifted_temp.isnull()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def repeated_matrix_reconstruction(num_pcs,num_iterations):\n",
    "\n",
    "    SVD = TruncatedSVD(n_components=num_pcs,random_state=42)\n",
    "    for i in range(num_iterations):\n",
    "        SVD.fit(movie_shifted)\n",
    "        #For the ease of applying masks we work with pandas\n",
    "        movie_represented =  pd.DataFrame(SVD.inverse_transform(SVD.transform(movie_shifted)),columns=movie_shifted.columns,index=movie_shifted.index)\n",
    "        loss = mean_squared_error(movie_represented[mask].fillna(0),movie_shifted_temp[mask].fillna(0))\n",
    "        #To just update the non-zero values of movie_reprented values to the true ratings\n",
    "        movie_represented[mask] = movie_shifted_temp[mask]\n",
    "    \n",
    "    #Mean shifting it  back\n",
    "    movie_represented = movie_represented + movie_means\n",
    "    movie_represented = movie_represented.clip(lower=0.5,upper=5)\n",
    "    return movie_represented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# iteration = 0\n",
    "# mse_last = 999\n",
    "# while iteration<10:\n",
    "#     iteration += 1\n",
    "#     svd = TruncatedSVD(n_components=20,random_state=42)\n",
    "#     svd.fit(mz)\n",
    "#     mzsvd = pd.DataFrame(svd.inverse_transform(svd.transform(mz)),columns=mz.columns,index=mz.index)\n",
    "\n",
    "#     mse = mean_squared_error(mzsvd[mask].fillna(0),mzm[mask].fillna(0))\n",
    "#     print('%i %.5f %.5f'%(iteration,mse,mse_last-mse))\n",
    "#     mzsvd[mask] = mzm[mask]\n",
    "\n",
    "#     mz = mzsvd\n",
    "#     if mse_last-mse<0.00001: break\n",
    "#     mse_last = mse\n",
    "\n",
    "# m_20 = mz+movie_means\n",
    "# m_20 = m_20.clip(lower=0.5,upper=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0.02925 998.97075\n",
      "2 0.02905 0.00020\n",
      "3 0.02890 0.00015\n",
      "4 0.02878 0.00012\n",
      "5 0.02867 0.00010\n",
      "6 0.02858 0.00009\n",
      "7 0.02850 0.00008\n",
      "8 0.02843 0.00007\n",
      "9 0.02836 0.00007\n",
      "10 0.02830 0.00006\n"
     ]
    }
   ],
   "source": [
    "# iteration = 0\n",
    "# mse_last = 999\n",
    "# while iteration<10:\n",
    "#     iteration += 1\n",
    "#     svd = TruncatedSVD(n_components=15,random_state=42)\n",
    "#     svd.fit(mz)\n",
    "#     mzsvd = pd.DataFrame(svd.inverse_transform(svd.transform(mz)),columns=mz.columns,index=mz.index)\n",
    "\n",
    "#     mse = mean_squared_error(mzsvd[mask].fillna(0),mzm[mask].fillna(0))\n",
    "#     print('%i %.5f %.5f'%(iteration,mse,mse_last-mse))\n",
    "#     mzsvd[mask] = mzm[mask]\n",
    "\n",
    "#     mz = mzsvd\n",
    "#     if mse_last-mse<0.00001: break\n",
    "#     mse_last = mse\n",
    "\n",
    "# m_15 = mz+movie_means\n",
    "# m_15 = m_15.clip(lower=0.5,upper=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#With 15 PCs\n",
    "representative_matrix_15 = repeated_matrix_reconstruction(15,10)\n",
    "#With 20 PCs\n",
    "representative_matrix_20 = repeated_matrix_reconstruction(20,10)\n",
    "\n",
    "\n",
    "#bagging\n",
    "representative_matrix = (representative_matrix_15+representative_matrix_20)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_best_tillnow = pd.read_csv(\"./ensemble_models2/submission_emsembling_8114_withval.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/pandas/core/indexing.py:189: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "svd_prediction['rating'] = svd_prediction.apply(lambda x:representative_matrix[representative_matrix.index==representative_matrix.userId][representative_matrix.movieId].values[0],axis=1)\n",
    "\n",
    "# There are some movies who did not have enough info to make prediction, so just used average value for user\n",
    "# missing = np.where(test.rating.isnull())[0]\n",
    "# test.ix[missing,'rating'] = user_means[test.loc[missing].userId].values\n",
    "\n",
    "indices=np.where(svd_prediction.rating.isna()) \n",
    "svd_prediction.rating.iloc[indices] = df_best_tillnow.Prediction.iloc[indices]\n",
    "\n",
    "rating_trunc = svd_prediction.rating\n",
    "\n",
    "df_test=pd.read_csv(data_folder+'test.csv') # Large 500MB\n",
    "predictions = np.zeros(len(df_test))\n",
    "for i in range(len(df_test)):\n",
    "    userid =  df_test.iloc[i,0]\n",
    "    movieid = df_test.iloc[i,1]\n",
    "    rating = PRED[i]\n",
    "    if rating>5:\n",
    "        rating = 5\n",
    "    if rating<0.5:\n",
    "        rating =0.5\n",
    "    predictions[i] = rating\n",
    "df_submission = pd.read_csv(data_folder+'dummy_submission.csv')\n",
    "df_submission.Prediction = predictions\n",
    "df_submission.to_csv('./ensemble_models2/Submission_truncSVD20_15_with_val.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_trunc = pd.read_csv(\"./ensemble_models2/Submission_truncSVD20_15_with_val.csv\")\n",
    "df_latent = pd.read_csv(\"./ensemble_models2/Submission_pq_with_val.csv\")\n",
    "df_regression = pd.read_csv(\"./ensemble_models2/Submission_regression_final.csv\")\n",
    "df_nn =  pd.read_csv(\"./ensemble_models2/Submission_regression_NN.csv\")\n",
    "\n",
    "\n",
    "PRED = (8*df_trunc.Prediction + df_latent.Prediction + df_regression.Prediction + 4*df_nn.Prediction)/14\n",
    "PRED = np.around(PRED,1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test=pd.read_csv(data_folder+'test.csv') # Large 500MB\n",
    "\n",
    "predictions = np.zeros(len(df_test))\n",
    "for i in range(len(df_test)):\n",
    "    userid =  df_test.iloc[i,0]\n",
    "    movieid = df_test.iloc[i,1]\n",
    "    rating = PRED[i]\n",
    "    if rating>5:\n",
    "        rating = 5\n",
    "    if rating<0.5:\n",
    "        rating =0.5\n",
    "    predictions[i] = rating\n",
    "df_submission = pd.read_csv(data_folder+'dummy_submission.csv')\n",
    "df_submission.Prediction = predictions\n",
    "df_submission.to_csv('./ensemble_models2/submission_emsembling_8114_withval_new.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}