Predict Doctor Consultation Fee.ipynb

{"cells":[{"metadata":{"trusted":true,"_uuid":"581fc77f5ca382c167ca97908228cd6370befc05"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\nimport re\nwarnings.filterwarnings('ignore')","execution_count":1,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab46fee75c1d2ffba761cc51b69dc196cb67dffd"},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":2,"outputs":[{"output_type":"stream","text":"['Final_Train.xlsx', 'Final_Test.xlsx']\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"2051644e6a38ccd345c29da588473e563312c03e"},"cell_type":"code","source":"train = pd.read_excel('../input/Final_Train.xlsx')\ntest = pd.read_excel('../input/Final_Test.xlsx')","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fc2d27a1a7813960efccbf88e912a22475951c6"},"cell_type":"code","source":"train.head()","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"                    Qualification ...  Fees\n0           BHMS, MD - Homeopathy ...   100\n1    BAMS, MD - Ayurveda Medicine ...   350\n2  MBBS, MS - Otorhinolaryngology ...   300\n3             BSc - Zoology, BAMS ...   250\n4                            BAMS ...   250\n\n[5 rows x 7 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Qualification</th>\n      <th>Experience</th>\n      <th>Rating</th>\n      <th>Place</th>\n      <th>Profile</th>\n      <th>Miscellaneous_Info</th>\n      <th>Fees</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>BHMS, MD - Homeopathy</td>\n      <td>24 years experience</td>\n      <td>100%</td>\n      <td>Kakkanad, Ernakulam</td>\n      <td>Homeopathy</td>\n      <td>100% 16 Feedback Kakkanad, Ernakulam</td>\n      <td>100</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>BAMS, MD - Ayurveda Medicine</td>\n      <td>12 years experience</td>\n      <td>98%</td>\n      <td>Whitefield, Bangalore</td>\n      <td>Ayurveda</td>\n      <td>98% 76 Feedback Whitefield, Bangalore</td>\n      <td>350</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>MBBS, MS - Otorhinolaryngology</td>\n      <td>9 years experience</td>\n      <td>NaN</td>\n      <td>Mathikere - BEL, Bangalore</td>\n      <td>ENT Specialist</td>\n      <td>NaN</td>\n      <td>300</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BSc - Zoology, BAMS</td>\n      <td>12 years experience</td>\n      <td>NaN</td>\n      <td>Bannerghatta Road, Bangalore</td>\n      <td>Ayurveda</td>\n      <td>Bannerghatta Road, Bangalore ₹250 Available on...</td>\n      <td>250</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>BAMS</td>\n      <td>20 years experience</td>\n      <td>100%</td>\n      <td>Keelkattalai, Chennai</td>\n      <td>Ayurveda</td>\n      <td>100% 4 Feedback Keelkattalai, Chennai</td>\n      <td>250</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"4dc48d75e4f2af058a9a166696cb5ac06cea5a2b"},"cell_type":"code","source":"train.info()","execution_count":5,"outputs":[{"output_type":"stream","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 5961 entries, 0 to 5960\nData columns (total 7 columns):\nQualification         5961 non-null object\nExperience            5961 non-null object\nRating                2659 non-null object\nPlace                 5936 non-null object\nProfile               5961 non-null object\nMiscellaneous_Info    3341 non-null object\nFees                  5961 non-null int64\ndtypes: int64(1), object(6)\nmemory usage: 326.1+ KB\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"a4c0ed787ff1c54c1c2e070fb7f7e3d8f73539e0"},"cell_type":"code","source":"print('Qualification:', train['Qualification'].nunique())\nprint('Experience:', train['Experience'].nunique())\nprint('Rating:', train['Rating'].nunique())\nprint('Place:', train['Place'].nunique())\nprint('Profile', train['Profile'].nunique())","execution_count":6,"outputs":[{"output_type":"stream","text":"Qualification: 1420\nExperience: 64\nRating: 51\nPlace: 877\nProfile 6\n","name":"stdout"}]},{"metadata":{"_uuid":"a1e0cebbdbb37954bdf9a81b1943b4b0f6647a17"},"cell_type":"markdown","source":"#### Data Pre-processing"},{"metadata":{"trusted":true,"_uuid":"639ef159f0f964b60c816fe3c94010c427f3b56a"},"cell_type":"code","source":"df_train = train[['Qualification', 'Profile', 'Experience', 'Place', 'Miscellaneous_Info', 'Rating','Fees']]\ndf_test = test[['Qualification', 'Profile','Experience', 'Place', 'Miscellaneous_Info', 'Rating']]","execution_count":31,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f30679103a2c936b114bd8e8f2198ca3d7de4eef"},"cell_type":"code","source":"df_train.head()","execution_count":32,"outputs":[{"output_type":"execute_result","execution_count":32,"data":{"text/plain":"                    Qualification         Profile ...  Rating Fees\n0           BHMS, MD - Homeopathy      Homeopathy ...    100%  100\n1    BAMS, MD - Ayurveda Medicine        Ayurveda ...     98%  350\n2  MBBS, MS - Otorhinolaryngology  ENT Specialist ...     NaN  300\n3             BSc - Zoology, BAMS        Ayurveda ...     NaN  250\n4                            BAMS        Ayurveda ...    100%  250\n\n[5 rows x 7 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Qualification</th>\n      <th>Profile</th>\n      <th>Experience</th>\n      <th>Place</th>\n      <th>Miscellaneous_Info</th>\n      <th>Rating</th>\n      <th>Fees</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>BHMS, MD - Homeopathy</td>\n      <td>Homeopathy</td>\n      <td>24 years experience</td>\n      <td>Kakkanad, Ernakulam</td>\n      <td>100% 16 Feedback Kakkanad, Ernakulam</td>\n      <td>100%</td>\n      <td>100</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>BAMS, MD - Ayurveda Medicine</td>\n      <td>Ayurveda</td>\n      <td>12 years experience</td>\n      <td>Whitefield, Bangalore</td>\n      <td>98% 76 Feedback Whitefield, Bangalore</td>\n      <td>98%</td>\n      <td>350</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>MBBS, MS - Otorhinolaryngology</td>\n      <td>ENT Specialist</td>\n      <td>9 years experience</td>\n      <td>Mathikere - BEL, Bangalore</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>300</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BSc - Zoology, BAMS</td>\n      <td>Ayurveda</td>\n      <td>12 years experience</td>\n      <td>Bannerghatta Road, Bangalore</td>\n      <td>Bannerghatta Road, Bangalore ₹250 Available on...</td>\n      <td>NaN</td>\n      <td>250</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>BAMS</td>\n      <td>Ayurveda</td>\n      <td>20 years experience</td>\n      <td>Keelkattalai, Chennai</td>\n      <td>100% 4 Feedback Keelkattalai, Chennai</td>\n      <td>100%</td>\n      <td>250</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"e348ad2c2bbc3ca979aae568c4f4df40bd65c18a"},"cell_type":"code","source":"df_train['Experience'] = df_train['Experience'].str.replace('years experience','').astype(int)\ndf_test['Experience'] = df_test['Experience'].str.replace('years experience','').astype(int)","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cb4ed32ace687b85a7ec84b7edfc1ccfd9baa24"},"cell_type":"code","source":"def clean_text(text):\n    text = str(text).lower()\n    text = re.sub(r'[^a-z]', ' ', text)\n    text = re.sub(r'\\s+', ' ', text)\n    return text\n\ndf_train['Qualification'] = df_train['Qualification'].apply(clean_text)\ndf_test['Qualification'] = df_test['Qualification'].apply(clean_text)\n\ndef clean_place(text):\n    text = str(text).lower()\n    text = re.sub(r'[^a-z0-9]', ' ', text)\n    text = re.sub(r'\\s+', ' ', text)\n    return text\n\ndf_train['Place'].fillna('missing', inplace=True)\ndf_test['Place'].fillna('missing', inplace=True)\ndf_train['Place'] = df_train['Place'].apply(clean_place)\ndf_test['Place'] = df_test['Place'].apply(clean_place)","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c71caa50abef7246b075590e0eff50d53054d6c5"},"cell_type":"code","source":"df_train['Rating'].fillna('0%', inplace=True)\ndf_test['Rating'].fillna('0%', inplace=True)\n\ndef clean_rating(text):\n    text = re.sub(r'%', '', str(text))\n    return text\n\ndf_train['Rating'] = df_train['Rating'].apply(clean_rating)\ndf_train['Rating'] = df_train['Rating'].astype(int)\n\ndf_test['Rating'] = df_test['Rating'].apply(clean_rating)\ndf_test['Rating'] = df_test['Rating'].astype(int)","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"966234ad2ed6672b2f7177db9055b7614341d378"},"cell_type":"code","source":"df_train['Miscellaneous_Info'].fillna('missing', inplace=True)\ndf_test['Miscellaneous_Info'].fillna('missing', inplace=True)\n\ndef get_feedback(feedback):\n    feedback = re.findall(r'\\d+ Feedback', str(feedback))\n    if feedback == []:\n        feedback = '0 Feedback'\n        return feedback\n    else:\n        return feedback[0]\n\ndf_train['Feedback'] = df_train['Miscellaneous_Info'].apply(get_feedback)\ndf_train['Feedback'] = df_train['Feedback'].str.replace(' Feedback','')\ndf_train['Feedback'] = df_train['Feedback'].astype(int)\n\ndf_test['Feedback'] = df_test['Miscellaneous_Info'].apply(get_feedback)\ndf_test['Feedback'] = df_test['Feedback'].str.replace(' Feedback','')\ndf_test['Feedback'] = df_test['Feedback'].astype(int)","execution_count":36,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"682c84d5bc7b6a253960fd765e9f382914a8873d"},"cell_type":"code","source":"def get_fee(text):\n    text = re.sub(r',', \"\", text)\n    text = re.findall(r'₹\\d+', text)\n    if text != []:\n        return text[0]\n    else:\n        return 0\n\ndf_train['Misc_Fees'] = df_train['Miscellaneous_Info'].apply(get_fee)\ndf_train['Misc_Fees'] = df_train['Misc_Fees'].str.replace('₹','')\ndf_train['Misc_Fees'].fillna(50, inplace=True)\ndf_train['Misc_Fees'] = df_train['Misc_Fees'].astype(int)\n\ndf_test['Misc_Fees'] = df_test['Miscellaneous_Info'].apply(get_fee)\ndf_test['Misc_Fees'] = df_test['Misc_Fees'].str.replace('₹','')\ndf_test['Misc_Fees'].fillna(50, inplace=True)\ndf_test['Misc_Fees'] = df_test['Misc_Fees'].astype(float)","execution_count":37,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb7c9a9280ebb2f27cca7b4097ad5150a932ce39"},"cell_type":"code","source":"def clean_misc(text):\n    text = str(text).lower()\n    text = re.sub(r'[^a-z0-9]', ' ', text)\n    text = re.sub(r'\\s+', ' ', text)\n    return text\n\ndf_train['Miscellaneous_Info'] = df_train['Miscellaneous_Info'].apply(clean_misc)\ndf_test['Miscellaneous_Info'] = df_test['Miscellaneous_Info'].apply(clean_misc)","execution_count":38,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d90e8732b10e9a1f504a2d88c9717ad37da5648b"},"cell_type":"code","source":"df_train = pd.get_dummies(df_train, columns=['Profile'])\ndf_test  = pd.get_dummies(df_test,  columns=['Profile'])","execution_count":39,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43f452451df2630c105190955625c5a74010c258"},"cell_type":"code","source":"df_train.head()","execution_count":40,"outputs":[{"output_type":"execute_result","execution_count":40,"data":{"text/plain":"                 Qualification         ...          Profile_Homeopathy\n0           bhms md homeopathy         ...                           1\n1    bams md ayurveda medicine         ...                           0\n2  mbbs ms otorhinolaryngology         ...                           0\n3             bsc zoology bams         ...                           0\n4                         bams         ...                           0\n\n[5 rows x 14 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Qualification</th>\n      <th>Experience</th>\n      <th>Place</th>\n      <th>Miscellaneous_Info</th>\n      <th>Rating</th>\n      <th>Fees</th>\n      <th>Feedback</th>\n      <th>Misc_Fees</th>\n      <th>Profile_Ayurveda</th>\n      <th>Profile_Dentist</th>\n      <th>Profile_Dermatologists</th>\n      <th>Profile_ENT Specialist</th>\n      <th>Profile_General Medicine</th>\n      <th>Profile_Homeopathy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bhms md homeopathy</td>\n      <td>24</td>\n      <td>kakkanad ernakulam</td>\n      <td>100 16 feedback kakkanad ernakulam</td>\n      <td>100</td>\n      <td>100</td>\n      <td>16</td>\n      <td>50</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>bams md ayurveda medicine</td>\n      <td>12</td>\n      <td>whitefield bangalore</td>\n      <td>98 76 feedback whitefield bangalore</td>\n      <td>98</td>\n      <td>350</td>\n      <td>76</td>\n      <td>50</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mbbs ms otorhinolaryngology</td>\n      <td>9</td>\n      <td>mathikere bel bangalore</td>\n      <td>missing</td>\n      <td>0</td>\n      <td>300</td>\n      <td>0</td>\n      <td>50</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>bsc zoology bams</td>\n      <td>12</td>\n      <td>bannerghatta road bangalore</td>\n      <td>bannerghatta road bangalore 250 available on s...</td>\n      <td>0</td>\n      <td>250</td>\n      <td>0</td>\n      <td>250</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>bams</td>\n      <td>20</td>\n      <td>keelkattalai chennai</td>\n      <td>100 4 feedback keelkattalai chennai</td>\n      <td>100</td>\n      <td>250</td>\n      <td>4</td>\n      <td>50</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"_uuid":"6e46fb0e846de8fb0f2306a963e55994d6fdc4d3"},"cell_type":"markdown","source":"#### train test split"},{"metadata":{"trusted":true,"_uuid":"b5f447dd32e2bd76a6a4b79cc2c5da79acf7062e"},"cell_type":"code","source":"X = df_train.drop(labels=['Fees'], axis=1)\ny = df_train['Fees'].values\n\nfrom sklearn.model_selection import train_test_split\nX_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.25, random_state=1)","execution_count":41,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7baf728114e7f6d8353c78e0ce74f1a52413a602"},"cell_type":"code","source":"X_train.shape, y_train.shape, X_cv.shape, y_cv.shape","execution_count":42,"outputs":[{"output_type":"execute_result","execution_count":42,"data":{"text/plain":"((4470, 13), (4470,), (1491, 13), (1491,))"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"0ccdd19532301b5fd8c67ba335c82dd4ba80ee4f"},"cell_type":"code","source":"X_train.columns","execution_count":43,"outputs":[{"output_type":"execute_result","execution_count":43,"data":{"text/plain":"Index(['Qualification', 'Experience', 'Place', 'Miscellaneous_Info', 'Rating',\n       'Feedback', 'Misc_Fees', 'Profile_Ayurveda', 'Profile_Dentist',\n       'Profile_Dermatologists', 'Profile_ENT Specialist',\n       'Profile_General Medicine', 'Profile_Homeopathy'],\n      dtype='object')"},"metadata":{}}]},{"metadata":{"_uuid":"3e6279bc7383a4c1fa842dba3b108f26eb9ba576"},"cell_type":"markdown","source":"#### build model"},{"metadata":{"trusted":true,"_uuid":"2d9d598c02a0748471640678d18a8062fd0f25f8"},"cell_type":"code","source":"X_train_qual = X_train['Qualification']\nX_cv_qual = X_cv['Qualification']\n\nX_train_place = X_train['Place']\nX_cv_place = X_cv['Place']\n\nX_train_misc = X_train['Miscellaneous_Info']\nX_cv_misc = X_cv['Miscellaneous_Info']\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\ntf1 = TfidfVectorizer(ngram_range=(1,2), binary=True, token_pattern=r'\\w{3,}')\nX_train_qual = tf1.fit_transform(X_train_qual)\nX_cv_qual = tf1.transform(X_cv_qual)\n\ntf2 = TfidfVectorizer()\nX_train_place = tf2.fit_transform(X_train_place)\nX_cv_place = tf2.transform(X_cv_place)\n\ntf3 = TfidfVectorizer(token_pattern=r'\\w{1,}', ngram_range=(1,2))\nX_train_misc = tf3.fit_transform(X_train_misc)\nX_cv_misc = tf3.transform(X_cv_misc)","execution_count":44,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2841de8bdeafd4a72b6aa6e776d7a2b788ee7ccd"},"cell_type":"code","source":"X_train_exp = np.array(X_train['Experience']).reshape(-1,1)\nX_cv_exp = np.array(X_cv['Experience']).reshape(-1,1)\n\nX_train_feedback = np.array(X_train['Feedback']).reshape(-1,1)\nX_cv_feedback = np.array(X_cv['Feedback']).reshape(-1,1)\n\nX_train_rating = np.array(X_train['Rating']).reshape(-1,1)\nX_cv_rating = np.array(X_cv['Rating']).reshape(-1,1)\n\ncols = ['Profile_Dentist', 'Profile_Dermatologists', 'Profile_ENT Specialist', 'Profile_General Medicine',\n       'Profile_Homeopathy']\nX_train_prof = X_train[cols]\nX_cv_prof = X_cv[cols]\n\nX_train_miscfees = np.array(X_train['Misc_Fees']).reshape(-1,1)\nX_cv_miscfees = np.array(X_cv['Misc_Fees']).reshape(-1,1)","execution_count":45,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ca9ce382aa3772118c72e02a59dce1dd7a4ec2e"},"cell_type":"code","source":"from math import sqrt \nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error","execution_count":46,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0256f33dfad364b7ac808ca80af251261eb27cf1"},"cell_type":"code","source":"from scipy.sparse import hstack\nmerged_train = hstack((X_train_exp, X_train_qual, X_train_prof, X_train_place, X_train_rating, X_train_misc, X_train_miscfees))\nmerged_cv  = hstack((X_cv_exp, X_cv_qual, X_cv_prof, X_cv_place, X_cv_rating, X_cv_misc, X_cv_miscfees))","execution_count":23,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6099c6af678aad2d968d870c4e5912d908d3c1a5"},"cell_type":"code","source":"merged_train.shape, merged_cv.shape","execution_count":24,"outputs":[{"output_type":"execute_result","execution_count":24,"data":{"text/plain":"((4470, 8600), (1491, 8600))"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"00b8dbdd2af81cf9506c99a07543b2fff4894d4e"},"cell_type":"code","source":"from xgboost import XGBRegressor\nxgb = XGBRegressor(learning_rate=0.02, \n                   gamma=100, \n                   max_depth=25,  \n                   min_child_weight=1, \n                   max_delta_step=0, \n                   subsample=0.75,  \n                   colsample_bylevel=0.95,  \n                   colsample_bytree=0.70,  \n                   reg_lambda=1)\nxgb.fit(merged_train, y_train)\ny_pred1 = xgb.predict(merged_cv)\ny_pred1 = y_pred1.astype(int)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred1)))","execution_count":25,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5616672599747987\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"a8b8a5ac8bc66debd3acdeaa3f9fbcd0eb8092c6"},"cell_type":"code","source":"import lightgbm as lgb\ntrain_data = lgb.Dataset(merged_train, label=y_train)\ntest_data = lgb.Dataset(merged_cv, label=y_cv)\n\nparam = {'objective': 'regression',\n         'boosting': 'dart',\n         'num_iterations': 152,   \n         'learning_rate': 0.1,  \n         'num_leaves': 52, \n         'max_depth': 28, \n         'min_data_in_leaf': 3, \n         'feature_fraction': 0.66,  \n         'feature_fraction_seed': 10,\n         'drop_seed': 4,\n         'alpha': 100,\n         'max_bin': 7,\n         'min_data_in_bin': 45,   \n         'metric': 'l2_root'\n         }\n\nlgbm = lgb.train(params=param,\n                 verbose_eval=0,\n                 train_set=train_data,\n                 valid_sets=[test_data])\n\ny_pred2 = lgbm.predict(merged_cv)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred2)))","execution_count":27,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5595818887221015\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"b378e9dd706ac75f6c874b825547541a812c172b"},"cell_type":"code","source":"from sklearn.ensemble import BaggingRegressor\nbr = BaggingRegressor(base_estimator=None, \n                      n_estimators=80,  \n                      max_samples=1.0, \n                      max_features=1.0, \n                      bootstrap=True, \n                      bootstrap_features=True,\n                      oob_score=True,\n                      n_jobs=None, \n                      random_state=13, \n                      verbose=0)\nbr.fit(merged_train, y_train)\ny_pred5 = br.predict(merged_cv)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred5)))\nprint('RMSE:', sqrt(mean_squared_error(y_cv, y_pred5)))\n#     0.58019310689049","execution_count":30,"outputs":[{"output_type":"stream","text":"RMSLE: 0.58019310689049\nRMSE: 160.65973494010672\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"3cfea32f1213edd391ef45cd4aa88c1e9b44a817"},"cell_type":"code","source":"from scipy.sparse import hstack\nmerged_train = hstack((X_train_exp, X_train_feedback, X_train_qual, X_train_prof, X_train_place, X_train_rating, X_train_misc))\nmerged_cv  = hstack((X_cv_exp, X_cv_feedback, X_cv_qual, X_cv_prof, X_cv_place, X_cv_rating, X_cv_misc))","execution_count":47,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adc8c486c690e97129d2a2305945fab908d362d1"},"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\ngb = GradientBoostingRegressor(loss='lad', \n                               learning_rate=0.2,  \n                               random_state=10, \n                               n_estimators=92,   \n                               max_depth=11,  \n                               subsample=1.0, \n                               min_samples_split=40, \n                               min_samples_leaf=1,\n                               max_features='auto')\ngb.fit(merged_train, y_train)\ny_pred3 = gb.predict(merged_cv)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred3)))","execution_count":48,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5812561135063448\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"c7eade7dd37a3073b8a418d454544a01abffc345"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nrf = RandomForestRegressor(n_estimators=29,  \n                           criterion='mse', \n                           max_depth=58,  \n                           min_samples_split=5, \n                           min_samples_leaf=2, \n                           min_weight_fraction_leaf=0.0, \n                           max_features='auto', \n                           max_leaf_nodes=None, \n                           min_impurity_decrease=0.20,  \n                           bootstrap=True, \n                           oob_score=True, \n                           n_jobs=-1, \n                           random_state=11)  \nrf.fit(merged_train, y_train)\ny_pred4 = rf.predict(merged_cv)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred4))) ","execution_count":49,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5928716390933947\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"8100df6e653bf2b9eb96db3e5b849c7d1c91490e"},"cell_type":"code","source":"y_predx = y_pred1*.4 + y_pred2*0.5 + y_pred3*0.1\ny_predx = y_predx.astype(int)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_predx)))","execution_count":50,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5553105196154928\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"fbd551f29eb0ebd616dd33560a266956ac3b2fd0"},"cell_type":"code","source":"y_predy = y_pred1*0.40 + y_pred2*0.45 + y_pred3*0.10 + y_pred5*0.05\ny_predy = y_predy.astype(int)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_predy)))","execution_count":51,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5556212301797688\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"017852befc54cd6e56013bfda34b306ac7b5bbcc"},"cell_type":"code","source":"y_pred = y_predx*.98 + y_predy*0.02\ny_pred = y_pred.astype(int)\nprint('RMSLE:', sqrt(mean_squared_log_error(y_cv, y_pred)))","execution_count":52,"outputs":[{"output_type":"stream","text":"RMSLE: 0.5552371830194157\n","name":"stdout"}]},{"metadata":{"_uuid":"6d66afc23d1c8d4ba777d138b012973590be53fa"},"cell_type":"markdown","source":"#### Now, predict on test set"},{"metadata":{"trusted":true,"_uuid":"213ebe92adce1bf1fb9dc1e729cbaa701c060f56"},"cell_type":"code","source":"X_train = df_train.drop(labels='Fees', axis=1)\ny_train = df_train['Fees'].values\n\nX_test = df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ed685415eb99f411f0c1074fcbdbfc5843c8ee9"},"cell_type":"code","source":"X_train_qual = X_train['Qualification']\nX_test_qual = X_test['Qualification']\n\nX_train_place = X_train['Place']\nX_test_place = X_test['Place']\n\nX_train_misc = X_train['Miscellaneous_Info']\nX_test_misc = X_test['Miscellaneous_Info']\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\ntf1 = TfidfVectorizer(ngram_range=(1,2), binary=True, token_pattern=r'\\w{3,}')\nX_train_qual = tf1.fit_transform(X_train_qual)\nX_test_qual = tf1.transform(X_test_qual)\n\ntf3 = TfidfVectorizer()\nX_train_place = tf3.fit_transform(X_train_place)\nX_test_place = tf3.transform(X_test_place)\n\ntf4 = TfidfVectorizer(token_pattern=r'\\w{1,}', ngram_range=(1,2))\nX_train_misc = tf4.fit_transform(X_train_misc)\nX_test_misc = tf4.transform(X_test_misc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c243b5fb84386e4abf827fb5cf6e39546bdd75f6"},"cell_type":"code","source":"X_train_exp = np.array(X_train['Experience']).reshape(-1,1)\nX_test_exp = np.array(X_test['Experience']).reshape(-1,1)\n\nX_train_feedback = np.array(X_train['Feedback']).reshape(-1,1)\nX_test_feedback = np.array(X_test['Feedback']).reshape(-1,1)\n\nX_train_rating = np.array(X_train['Rating']).reshape(-1,1)\nX_test_rating = np.array(X_test['Rating']).reshape(-1,1)\n\ncols = ['Profile_Dentist', 'Profile_Dermatologists', 'Profile_ENT Specialist', 'Profile_General Medicine',\n       'Profile_Homeopathy']\nX_train_prof = X_train[cols]\nX_test_prof = X_test[cols]\n\nX_train_miscfees = np.array(X_train['Misc_Fees']).reshape(-1,1)\nX_test_miscfees = np.array(X_test['Misc_Fees']).reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"965b4ab6f029f95f085106cf19fd2c132b5470ef"},"cell_type":"code","source":"from scipy.sparse import hstack\nmerged_train = hstack((X_train_exp, X_train_qual, X_train_prof, X_train_place, X_train_rating, X_train_misc, X_train_miscfees))\nmerged_test  = hstack((X_test_exp, X_test_qual, X_test_prof, X_test_place, X_test_rating, X_test_misc, X_test_miscfees))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90aa97035ad63f20b07a95c7c5d0d52d90609384"},"cell_type":"code","source":"merged_train.shape, merged_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ab74fcfed4463bb08360ccd336b552ba22af900"},"cell_type":"code","source":"from xgboost import XGBRegressor\nxgb = XGBRegressor(learning_rate=0.02, \n                   gamma=100, \n                   max_depth=25,  \n                   min_child_weight=1, \n                   max_delta_step=0, \n                   subsample=0.75,  \n                   colsample_bylevel=0.95,  \n                   colsample_bytree=0.70,  \n                   reg_lambda=1)\nxgb.fit(merged_train, y_train)\ny_pred1 = xgb.predict(merged_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ed8adacf9a32fc93c078253e5b445923400fca8"},"cell_type":"code","source":"import lightgbm as lgb\ntrain_data = lgb.Dataset(merged_train, label=y_train)\n\nparam = {'objective': 'regression',\n         'boosting': 'dart',\n         'num_iterations': 137,   \n         'learning_rate': 0.1,\n         'num_leaves': 52,   \n         'max_depth': 33,   \n         'min_data_in_leaf': 3,  \n         'feature_fraction': 0.54,  \n         'feature_fraction_seed': 50,  \n         'alpha': 100,\n         'max_bin': 7,\n         'min_data_in_bin': 45,   \n         'metric': 'l2_root'\n         }\n\nlgbm = lgb.train(params=param,\n                 train_set=train_data)\n\ny_pred2 = lgbm.predict(merged_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cb7b732a39c8f572e9480f964154e79b3831456"},"cell_type":"code","source":"from sklearn.ensemble import BaggingRegressor\nbr = BaggingRegressor(base_estimator=None, \n                      n_estimators=80,  \n                      max_samples=1.0, \n                      max_features=1.0, \n                      bootstrap=True, \n                      bootstrap_features=True,\n                      oob_score=True,\n                      n_jobs=None, \n                      random_state=13, #4\n                      verbose=0)\nbr.fit(merged_train, y_train)\ny_pred5 = br.predict(merged_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa4c57a57a65f513eca98aeb8dd8af7e25ef5e4a"},"cell_type":"code","source":"from scipy.sparse import hstack\nmerged_train = hstack((X_train_exp, X_train_feedback, X_train_qual, X_train_prof, X_train_place, X_train_rating, X_train_misc))\nmerged_test  = hstack((X_test_exp, X_test_feedback, X_test_qual, X_test_prof, X_test_place, X_test_rating, X_test_misc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd85de8d77658788c05983c2189d7eacf8d45797"},"cell_type":"code","source":"merged_train.shape, merged_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28d4b284ef0b0afdc2fafcb085e25d3aea88ea9a"},"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\ngb = GradientBoostingRegressor(loss='lad', \n                               learning_rate=0.2, #0.2\n                               random_state=10, \n                               n_estimators=92,   \n                               max_depth=11, #11, \n                               subsample=1.0, \n                               min_samples_split=40, \n                               min_samples_leaf=1,\n                               max_features='auto')\ngb.fit(merged_train, y_train)\ny_pred3 = gb.predict(merged_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af50aee202aa2ea061a11e29d290e109a4a1d7a"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nrf = RandomForestRegressor(n_estimators=29, #25, 29\n                           criterion='mse', \n                           max_depth=58, #55, 58\n                           min_samples_split=5, \n                           min_samples_leaf=2, \n                           min_weight_fraction_leaf=0.0, \n                           max_features='auto', \n                           max_leaf_nodes=None, \n                           min_impurity_decrease=0.20, #0.20, \n                           bootstrap=True, \n                           oob_score=True, \n                           n_jobs=-1, \n                           random_state=11) \nrf.fit(merged_train, y_train)\ny_pred4 = rf.predict(merged_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88d3baa1a0d6c9ba6485ed21ecd61f377a3896f3"},"cell_type":"code","source":"y_pred1.shape, y_pred2.shape, y_pred3.shape, y_pred4.shape, y_pred5.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da45808769fed3c3957c266f6601783f2aab56a6"},"cell_type":"code","source":"y_predx = y_pred1*.4 + y_pred2*0.5 + y_pred3*0.10\ny_predx = y_predx.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a4bb4167de6f15e22ebe104d91e1306b165fde"},"cell_type":"code","source":"y_predy = y_pred1*0.40 + y_pred2*0.45 + y_pred3*0.10 + y_pred5*0.05\ny_predy = y_predy.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e254731e0991d11967b9b92ef898db75d6820cb"},"cell_type":"code","source":"y_pred = y_predx*.98 + y_predy*0.02\ny_pred = y_pred.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc7f8df4c8acd61fb66d38fd3d16b44127a56190"},"cell_type":"markdown","source":"#### write predictions to output file"},{"metadata":{"trusted":true,"_uuid":"fe88dfef696e96f12650f0e53bc748fa8dcaf43e"},"cell_type":"code","source":"df_sub = pd.DataFrame(data=y_pred, columns=['Fees'])\nwriter = pd.ExcelWriter('output74.xlsx', engine='xlsxwriter')\ndf_sub.to_excel(writer,sheet_name='Sheet1', index=False)\nwriter.save()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":false,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{"height":"calc(100% - 180px)","left":"10px","top":"150px","width":"213.6px"},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}