05_dataset_preprocessing_after_rfe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 05 - Pre-processing After Removing Features with RFE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as sp\n",
    "from joblib import dump, load\n",
    "\n",
    "from utils import dataframe_utils,preprocess_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "           Patient ID  Patient age quantile SARS-Cov-2 exam result  \\\n0     44477f75e8169d2                    13               negative   \n1     126e9dd13932f68                    17               negative   \n2     a46b4402a0e5696                     8               negative   \n3     f7d619a94f97c45                     5               negative   \n4     d9e41465789c2b5                    15               negative   \n...               ...                   ...                    ...   \n5639  ae66feb9e4dc3a0                     3               positive   \n5640  517c2834024f3ea                    17               negative   \n5641  5c57d6037fe266d                     4               negative   \n5642  c20c44766f28291                    10               negative   \n5643  2697fdccbfeb7f7                    19               positive   \n\n      Patient addmited to regular ward (1=yes, 0=no)  \\\n0                                                  0   \n1                                                  0   \n2                                                  0   \n3                                                  0   \n4                                                  0   \n...                                              ...   \n5639                                               0   \n5640                                               0   \n5641                                               0   \n5642                                               0   \n5643                                               0   \n\n      Patient addmited to semi-intensive unit (1=yes, 0=no)  \\\n0                                                     0       \n1                                                     0       \n2                                                     0       \n3                                                     0       \n4                                                     0       \n...                                                 ...       \n5639                                                  0       \n5640                                                  0       \n5641                                                  0       \n5642                                                  0       \n5643                                                  0       \n\n      Patient addmited to intensive care unit (1=yes, 0=no)  Hematocrit  \\\n0                                                     0             NaN   \n1                                                     0        0.236515   \n2                                                     0             NaN   \n3                                                     0             NaN   \n4                                                     0             NaN   \n...                                                 ...             ...   \n5639                                                  0             NaN   \n5640                                                  0             NaN   \n5641                                                  0             NaN   \n5642                                                  0             NaN   \n5643                                                  0        0.694287   \n\n      Hemoglobin  Platelets  Mean platelet volume   ...  \\\n0            NaN        NaN                    NaN  ...   \n1      -0.022340  -0.517413               0.010677  ...   \n2            NaN        NaN                    NaN  ...   \n3            NaN        NaN                    NaN  ...   \n4            NaN        NaN                    NaN  ...   \n...          ...        ...                    ...  ...   \n5639         NaN        NaN                    NaN  ...   \n5640         NaN        NaN                    NaN  ...   \n5641         NaN        NaN                    NaN  ...   \n5642         NaN        NaN                    NaN  ...   \n5643    0.541564  -0.906829              -0.325903  ...   \n\n      Hb saturation (arterial blood gases)  \\\n0                                      NaN   \n1                                      NaN   \n2                                      NaN   \n3                                      NaN   \n4                                      NaN   \n...                                    ...   \n5639                                   NaN   \n5640                                   NaN   \n5641                                   NaN   \n5642                                   NaN   \n5643                                   NaN   \n\n      pCO2 (arterial blood gas analysis)  \\\n0                                    NaN   \n1                                    NaN   \n2                                    NaN   \n3                                    NaN   \n4                                    NaN   \n...                                  ...   \n5639                                 NaN   \n5640                                 NaN   \n5641                                 NaN   \n5642                                 NaN   \n5643                                 NaN   \n\n      Base excess (arterial blood gas analysis)  \\\n0                                           NaN   \n1                                           NaN   \n2                                           NaN   \n3                                           NaN   \n4                                           NaN   \n...                                         ...   \n5639                                        NaN   \n5640                                        NaN   \n5641                                        NaN   \n5642                                        NaN   \n5643                                        NaN   \n\n      pH (arterial blood gas analysis)  \\\n0                                  NaN   \n1                                  NaN   \n2                                  NaN   \n3                                  NaN   \n4                                  NaN   \n...                                ...   \n5639                               NaN   \n5640                               NaN   \n5641                               NaN   \n5642                               NaN   \n5643                               NaN   \n\n      Total CO2 (arterial blood gas analysis)  \\\n0                                         NaN   \n1                                         NaN   \n2                                         NaN   \n3                                         NaN   \n4                                         NaN   \n...                                       ...   \n5639                                      NaN   \n5640                                      NaN   \n5641                                      NaN   \n5642                                      NaN   \n5643                                      NaN   \n\n      HCO3 (arterial blood gas analysis)  pO2 (arterial blood gas analysis)  \\\n0                                    NaN                                NaN   \n1                                    NaN                                NaN   \n2                                    NaN                                NaN   \n3                                    NaN                                NaN   \n4                                    NaN                                NaN   \n...                                  ...                                ...   \n5639                                 NaN                                NaN   \n5640                                 NaN                                NaN   \n5641                                 NaN                                NaN   \n5642                                 NaN                                NaN   \n5643                                 NaN                                NaN   \n\n      Arteiral Fio2  Phosphor  ctO2 (arterial blood gas analysis)  \n0               NaN       NaN                                 NaN  \n1               NaN       NaN                                 NaN  \n2               NaN       NaN                                 NaN  \n3               NaN       NaN                                 NaN  \n4               NaN       NaN                                 NaN  \n...             ...       ...                                 ...  \n5639            NaN       NaN                                 NaN  \n5640            NaN       NaN                                 NaN  \n5641            NaN       NaN                                 NaN  \n5642            NaN       NaN                                 NaN  \n5643            NaN       NaN                                 NaN  \n\n[5644 rows x 111 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Patient ID</th>\n      <th>Patient age quantile</th>\n      <th>SARS-Cov-2 exam result</th>\n      <th>Patient addmited to regular ward (1=yes, 0=no)</th>\n      <th>Patient addmited to semi-intensive unit (1=yes, 0=no)</th>\n      <th>Patient addmited to intensive care unit (1=yes, 0=no)</th>\n      <th>Hematocrit</th>\n      <th>Hemoglobin</th>\n      <th>Platelets</th>\n      <th>Mean platelet volume</th>\n      <th>...</th>\n      <th>Hb saturation (arterial blood gases)</th>\n      <th>pCO2 (arterial blood gas analysis)</th>\n      <th>Base excess (arterial blood gas analysis)</th>\n      <th>pH (arterial blood gas analysis)</th>\n      <th>Total CO2 (arterial blood gas analysis)</th>\n      <th>HCO3 (arterial blood gas analysis)</th>\n      <th>pO2 (arterial blood gas analysis)</th>\n      <th>Arteiral Fio2</th>\n      <th>Phosphor</th>\n      <th>ctO2 (arterial blood gas analysis)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>44477f75e8169d2</td>\n      <td>13</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>126e9dd13932f68</td>\n      <td>17</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.236515</td>\n      <td>-0.022340</td>\n      <td>-0.517413</td>\n      <td>0.010677</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>a46b4402a0e5696</td>\n      <td>8</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>f7d619a94f97c45</td>\n      <td>5</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>d9e41465789c2b5</td>\n      <td>15</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5639</th>\n      <td>ae66feb9e4dc3a0</td>\n      <td>3</td>\n      <td>positive</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5640</th>\n      <td>517c2834024f3ea</td>\n      <td>17</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5641</th>\n      <td>5c57d6037fe266d</td>\n      <td>4</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5642</th>\n      <td>c20c44766f28291</td>\n      <td>10</td>\n      <td>negative</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5643</th>\n      <td>2697fdccbfeb7f7</td>\n      <td>19</td>\n      <td>positive</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.694287</td>\n      <td>0.541564</td>\n      <td>-0.906829</td>\n      <td>-0.325903</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>5644 rows × 111 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "df_covid = pd.read_excel('datasets/dataset.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "['Patient ID',\n 'Patient age quantile',\n 'SARS-Cov-2 exam result',\n 'Patient addmited to regular ward (1=yes, 0=no)',\n 'Patient addmited to semi-intensive unit (1=yes, 0=no)',\n 'Patient addmited to intensive care unit (1=yes, 0=no)',\n 'Hematocrit',\n 'Hemoglobin',\n 'Platelets',\n 'Mean platelet volume ',\n 'Red blood Cells',\n 'Lymphocytes',\n 'Mean corpuscular hemoglobin concentration\\xa0(MCHC)',\n 'Leukocytes',\n 'Basophils',\n 'Mean corpuscular hemoglobin (MCH)',\n 'Eosinophils',\n 'Mean corpuscular volume (MCV)',\n 'Monocytes',\n 'Red blood cell distribution width (RDW)',\n 'Serum Glucose',\n 'Respiratory Syncytial Virus',\n 'Influenza A',\n 'Influenza B',\n 'Parainfluenza 1',\n 'CoronavirusNL63',\n 'Rhinovirus/Enterovirus',\n 'Mycoplasma pneumoniae',\n 'Coronavirus HKU1',\n 'Parainfluenza 3',\n 'Chlamydophila pneumoniae',\n 'Adenovirus',\n 'Parainfluenza 4',\n 'Coronavirus229E',\n 'CoronavirusOC43',\n 'Inf A H1N1 2009',\n 'Bordetella pertussis',\n 'Metapneumovirus',\n 'Parainfluenza 2',\n 'Neutrophils',\n 'Urea',\n 'Proteina C reativa mg/dL',\n 'Creatinine',\n 'Potassium',\n 'Sodium',\n 'Influenza B, rapid test',\n 'Influenza A, rapid test',\n 'Alanine transaminase',\n 'Aspartate transaminase',\n 'Gamma-glutamyltransferase\\xa0',\n 'Total Bilirubin',\n 'Direct Bilirubin',\n 'Indirect Bilirubin',\n 'Alkaline phosphatase',\n 'Ionized calcium\\xa0',\n 'Strepto A',\n 'Magnesium',\n 'pCO2 (venous blood gas analysis)',\n 'Hb saturation (venous blood gas analysis)',\n 'Base excess (venous blood gas analysis)',\n 'pO2 (venous blood gas analysis)',\n 'Fio2 (venous blood gas analysis)',\n 'Total CO2 (venous blood gas analysis)',\n 'pH (venous blood gas analysis)',\n 'HCO3 (venous blood gas analysis)',\n 'Rods #',\n 'Segmented',\n 'Promyelocytes',\n 'Metamyelocytes',\n 'Myelocytes',\n 'Myeloblasts',\n 'Urine - Esterase',\n 'Urine - Aspect',\n 'Urine - pH',\n 'Urine - Hemoglobin',\n 'Urine - Bile pigments',\n 'Urine - Ketone Bodies',\n 'Urine - Nitrite',\n 'Urine - Density',\n 'Urine - Urobilinogen',\n 'Urine - Protein',\n 'Urine - Sugar',\n 'Urine - Leukocytes',\n 'Urine - Crystals',\n 'Urine - Red blood cells',\n 'Urine - Hyaline cylinders',\n 'Urine - Granular cylinders',\n 'Urine - Yeasts',\n 'Urine - Color',\n 'Partial thromboplastin time\\xa0(PTT)\\xa0',\n 'Relationship (Patient/Normal)',\n 'International normalized ratio (INR)',\n 'Lactic Dehydrogenase',\n 'Prothrombin time (PT), Activity',\n 'Vitamin B12',\n 'Creatine phosphokinase\\xa0(CPK)\\xa0',\n 'Ferritin',\n 'Arterial Lactic Acid',\n 'Lipase dosage',\n 'D-Dimer',\n 'Albumin',\n 'Hb saturation (arterial blood gases)',\n 'pCO2 (arterial blood gas analysis)',\n 'Base excess (arterial blood gas analysis)',\n 'pH (arterial blood gas analysis)',\n 'Total CO2 (arterial blood gas analysis)',\n 'HCO3 (arterial blood gas analysis)',\n 'pO2 (arterial blood gas analysis)',\n 'Arteiral Fio2',\n 'Phosphor',\n 'ctO2 (arterial blood gas analysis)']"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "list(df_covid.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(5644, 19)"
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "source": [
    "df_covid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_covid = df_covid[[\n",
    "    'Patient age quantile', \n",
    "    'SARS-Cov-2 exam result', \n",
    "    'Platelets',\n",
    "    'Mean platelet volume ', \n",
    "    'Red blood Cells', \n",
    "    'Lymphocytes',\n",
    "    'Mean corpuscular hemoglobin concentration\\xa0(MCHC)', \n",
    "    'Leukocytes',\n",
    "    'Basophils', \n",
    "    'Mean corpuscular hemoglobin (MCH)', \n",
    "    'Eosinophils',\n",
    "    'Mean corpuscular volume (MCV)', \n",
    "    'Monocytes',\n",
    "    'Red blood cell distribution width (RDW)', \n",
    "    'Influenza B',\n",
    "    'CoronavirusNL63', \n",
    "    'Rhinovirus/Enterovirus', \n",
    "    'Coronavirus HKU1',\n",
    "    'Inf A H1N1 2009']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Patient age quantile                                5644\nSARS-Cov-2 exam result                              5644\nPlatelets                                            602\nMean platelet volume                                 599\nRed blood Cells                                      602\nLymphocytes                                          602\nMean corpuscular hemoglobin concentration (MCHC)     602\nLeukocytes                                           602\nBasophils                                            602\nMean corpuscular hemoglobin (MCH)                    602\nEosinophils                                          602\nMean corpuscular volume (MCV)                        602\nMonocytes                                            601\nRed blood cell distribution width (RDW)              602\nInfluenza B                                         1354\nCoronavirusNL63                                     1352\nRhinovirus/Enterovirus                              1352\nCoronavirus HKU1                                    1352\nInf A H1N1 2009                                     1352\ndtype: int64"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_counts = df_covid.count()\n",
    "display(df_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(362, 19)"
     },
     "metadata": {},
     "execution_count": 44
    }
   ],
   "source": [
    "df_covid_subset_final = df_covid.dropna()\n",
    "df_covid_subset_final.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even when taking the most relevant features from the original dataset, the result is pretty much the same size as the previous dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_by_data_threshold(df,threshold,axis):\n",
    "    print(f'threshold (%) = {threshold * 100}')\n",
    "    if axis == 'row': axis = 1\n",
    "    elif axis == 'col' or axis == 'column': axis = 0\n",
    "\n",
    "    max_values = df.shape[axis]\n",
    "    value_threshold = int(threshold * max_values)\n",
    "    mask_values_to_keep = df.count(axis = axis) >= value_threshold\n",
    "    print('threshold (data points) =',value_threshold)\n",
    "    if axis == 1: return df[mask_values_to_keep]\n",
    "    elif axis == 0: return df.loc[:,mask_values_to_keep.values]"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit ('puc': venv)",
   "language": "python",
   "name": "python37764bitpucvenv3c8e04841bf343089962c3369eba30a4"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 4
}