fairness_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "administrative-dutch",
   "metadata": {},
   "source": [
    "# Fairness Metrics\n",
    "\n",
    "This notebook implements the statistical fairness metrics from:\n",
    "*Towards the Right Kind of Fairness in AI* by Boris Ruf and Marcin Detyniecki (2021)\n",
    "https://arxiv.org/abs/2102.08453\n",
    "\n",
    "Example with the `german-risk-scoring.csv` dataset.\n",
    "\n",
    "Contributeurs : Xavier Lioneton & Francis Wolinski"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "apart-flood",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "southeast-treatment",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas.api.types import is_numeric_dtype\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "from IPython.display import display, Markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sensitive-opera",
   "metadata": {},
   "source": [
    "## Data Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "simplified-lover",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 21 columns):\n",
      " #   Column                                                    Non-Null Count  Dtype \n",
      "---  ------                                                    --------------  ----- \n",
      " 0   Status of existing checking account                       1000 non-null   object\n",
      " 1   Duration in month                                         1000 non-null   int64 \n",
      " 2   Credit history                                            1000 non-null   object\n",
      " 3   Purpose                                                   1000 non-null   object\n",
      " 4   Credit amount                                             1000 non-null   int64 \n",
      " 5   Savings account/bonds                                     1000 non-null   object\n",
      " 6   Present employment since                                  1000 non-null   object\n",
      " 7   Installment rate in percentage of disposable income       1000 non-null   int64 \n",
      " 8   Personal status and sex                                   1000 non-null   object\n",
      " 9   Other debtors / guarantors                                1000 non-null   object\n",
      " 10  Present residence since                                   1000 non-null   int64 \n",
      " 11  Property                                                  1000 non-null   object\n",
      " 12  Age in years                                              1000 non-null   int64 \n",
      " 13  Other installment plans                                   1000 non-null   object\n",
      " 14  Housing                                                   1000 non-null   object\n",
      " 15  Number of existing credits at this bank                   1000 non-null   int64 \n",
      " 16  Job                                                       1000 non-null   object\n",
      " 17  Number of people being liable to provide maintenance for  1000 non-null   int64 \n",
      " 18  Telephone                                                 1000 non-null   object\n",
      " 19  foreign worker                                            1000 non-null   object\n",
      " 20  Cost Matrix(Risk)                                         1000 non-null   object\n",
      "dtypes: int64(7), object(14)\n",
      "memory usage: 164.2+ KB\n"
     ]
    }
   ],
   "source": [
    "# dataset\n",
    "data = pd.read_csv('german-risk-scoring.csv')\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "orange-drink",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Good Risk    700\n",
       "Bad Risk     300\n",
       "Name: Cost Matrix(Risk), dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# target\n",
    "data['Cost Matrix(Risk)'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "practical-withdrawal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male:single                          548\n",
       "female:divorced/separated/married    310\n",
       "male:married/widowed                  92\n",
       "male:divorced/separated               50\n",
       "Name: Personal status and sex, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Personal status and sex\n",
    "data[\"Personal status and sex\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "final-anger",
   "metadata": {},
   "source": [
    "## Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "necessary-puzzle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sex column\n",
    "data[\"sex\"] = data[\"Personal status and sex\"].apply(lambda x : x.split(\":\")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "royal-wholesale",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create X=features, y=target\n",
    "X = data.drop(columns = 'Cost Matrix(Risk)')\n",
    "y = data['Cost Matrix(Risk)'].map({\"Good Risk\": 1, \"Bad Risk\": 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "external-commander",
   "metadata": {},
   "outputs": [],
   "source": [
    "# type modifications\n",
    "\n",
    "cols_cat = [\n",
    "    'Status of existing checking account',\n",
    "    'Credit history',\n",
    "    'Purpose',\n",
    "    'Savings account/bonds',\n",
    "    'Present employment since',\n",
    "    'Personal status and sex', \n",
    "    'Other debtors / guarantors',\n",
    "    'Property',\n",
    "    'Other installment plans',\n",
    "    'Housing',\n",
    "    'Job',\n",
    "    'Telephone',\n",
    "    'foreign worker',\n",
    "    'sex'\n",
    "       ]\n",
    "\n",
    "cols_num = [\n",
    "    'Duration in month',\n",
    "    'Credit amount',\n",
    "    'Installment rate in percentage of disposable income',\n",
    "    'Present residence since',\n",
    "    'Age in years',\n",
    "    'Number of existing credits at this bank',\n",
    "    'Number of people being liable to provide maintenance for',\n",
    "    ]\n",
    "\n",
    "for col in cols_cat:\n",
    "    data[col] = data[col].astype(str)\n",
    "    \n",
    "for col in cols_num:\n",
    "    data[col] = data[col].astype(float)\n",
    "\n",
    "cols = cols_cat + cols_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "essential-studio",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Status of existing checking account     4\n",
       "Credit history                          5\n",
       "Purpose                                10\n",
       "Savings account/bonds                   5\n",
       "Present employment since                5\n",
       "Personal status and sex                 4\n",
       "Other debtors / guarantors              3\n",
       "Property                                4\n",
       "Other installment plans                 3\n",
       "Housing                                 3\n",
       "Job                                     4\n",
       "Telephone                               2\n",
       "foreign worker                          2\n",
       "sex                                     2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# unique values of categorical columns\n",
    "\n",
    "X[cols_cat].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "heard-guess",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 63)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# all to numbers\n",
    "\n",
    "encoder = OneHotEncoder()\n",
    "X_cat = encoder.fit_transform(X[cols_cat]).toarray()\n",
    "X_num = X[cols_num]\n",
    "X_prep = np.concatenate((X_num, X_cat), axis=1)\n",
    "X_prep.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "reasonable-intro",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 63)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data prepared\n",
    "\n",
    "cols = data[cols_num].columns.tolist() + encoder.get_feature_names(input_features=X[cols_cat].columns).tolist()\n",
    "data_prep = pd.DataFrame(X_prep, columns=cols)\n",
    "data_prep.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "perceived-titanium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Duration in month</th>\n",
       "      <th>Credit amount</th>\n",
       "      <th>Installment rate in percentage of disposable income</th>\n",
       "      <th>Present residence since</th>\n",
       "      <th>Age in years</th>\n",
       "      <th>Number of existing credits at this bank</th>\n",
       "      <th>Number of people being liable to provide maintenance for</th>\n",
       "      <th>Status of existing checking account_0 &lt;= &lt;200 DM</th>\n",
       "      <th>Status of existing checking account_&lt;0 DM</th>\n",
       "      <th>Status of existing checking account_&gt;= 200 DM</th>\n",
       "      <th>...</th>\n",
       "      <th>Job_management/ highly qualified employee</th>\n",
       "      <th>Job_skilled employee / official</th>\n",
       "      <th>Job_unemployed/ unskilled  - non-resident</th>\n",
       "      <th>Job_unskilled - resident</th>\n",
       "      <th>Telephone_none</th>\n",
       "      <th>Telephone_yes</th>\n",
       "      <th>foreign worker_no</th>\n",
       "      <th>foreign worker_yes</th>\n",
       "      <th>sex_female</th>\n",
       "      <th>sex_male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.0</td>\n",
       "      <td>1169.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48.0</td>\n",
       "      <td>5951.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12.0</td>\n",
       "      <td>2096.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42.0</td>\n",
       "      <td>7882.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24.0</td>\n",
       "      <td>4870.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Duration in month  Credit amount  \\\n",
       "0                6.0         1169.0   \n",
       "1               48.0         5951.0   \n",
       "2               12.0         2096.0   \n",
       "3               42.0         7882.0   \n",
       "4               24.0         4870.0   \n",
       "\n",
       "   Installment rate in percentage of disposable income  \\\n",
       "0                                                4.0     \n",
       "1                                                2.0     \n",
       "2                                                2.0     \n",
       "3                                                2.0     \n",
       "4                                                3.0     \n",
       "\n",
       "   Present residence since  Age in years  \\\n",
       "0                      4.0          67.0   \n",
       "1                      2.0          22.0   \n",
       "2                      3.0          49.0   \n",
       "3                      4.0          45.0   \n",
       "4                      4.0          53.0   \n",
       "\n",
       "   Number of existing credits at this bank  \\\n",
       "0                                      2.0   \n",
       "1                                      1.0   \n",
       "2                                      1.0   \n",
       "3                                      1.0   \n",
       "4                                      2.0   \n",
       "\n",
       "   Number of people being liable to provide maintenance for  \\\n",
       "0                                                1.0          \n",
       "1                                                1.0          \n",
       "2                                                2.0          \n",
       "3                                                2.0          \n",
       "4                                                2.0          \n",
       "\n",
       "   Status of existing checking account_0 <= <200 DM  \\\n",
       "0                                               0.0   \n",
       "1                                               1.0   \n",
       "2                                               0.0   \n",
       "3                                               0.0   \n",
       "4                                               0.0   \n",
       "\n",
       "   Status of existing checking account_<0 DM  \\\n",
       "0                                        1.0   \n",
       "1                                        0.0   \n",
       "2                                        0.0   \n",
       "3                                        1.0   \n",
       "4                                        1.0   \n",
       "\n",
       "   Status of existing checking account_>= 200 DM   ...  \\\n",
       "0                                             0.0  ...   \n",
       "1                                             0.0  ...   \n",
       "2                                             0.0  ...   \n",
       "3                                             0.0  ...   \n",
       "4                                             0.0  ...   \n",
       "\n",
       "   Job_management/ highly qualified employee  Job_skilled employee / official  \\\n",
       "0                                        0.0                              1.0   \n",
       "1                                        0.0                              1.0   \n",
       "2                                        0.0                              0.0   \n",
       "3                                        0.0                              1.0   \n",
       "4                                        0.0                              1.0   \n",
       "\n",
       "   Job_unemployed/ unskilled  - non-resident  Job_unskilled - resident  \\\n",
       "0                                        0.0                       0.0   \n",
       "1                                        0.0                       0.0   \n",
       "2                                        0.0                       1.0   \n",
       "3                                        0.0                       0.0   \n",
       "4                                        0.0                       0.0   \n",
       "\n",
       "   Telephone_none  Telephone_yes  foreign worker_no  foreign worker_yes  \\\n",
       "0             0.0            1.0                0.0                 1.0   \n",
       "1             1.0            0.0                0.0                 1.0   \n",
       "2             1.0            0.0                0.0                 1.0   \n",
       "3             1.0            0.0                0.0                 1.0   \n",
       "4             1.0            0.0                0.0                 1.0   \n",
       "\n",
       "   sex_female  sex_male  \n",
       "0         0.0       1.0  \n",
       "1         1.0       0.0  \n",
       "2         0.0       1.0  \n",
       "3         0.0       1.0  \n",
       "4         0.0       1.0  \n",
       "\n",
       "[5 rows x 63 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data prepared\n",
    "\n",
    "data_prep.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promising-divorce",
   "metadata": {},
   "source": [
    "## Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "boxed-flexibility",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(800, 63) (200, 63) (800,) (200,)\n"
     ]
    }
   ],
   "source": [
    "# split train test\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_prep, y, test_size=0.2, random_state=42)\n",
    "X_train = X_train.copy()\n",
    "X_test = X_test.copy()\n",
    "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0b5d418",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "useful-aaron",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(max_iter=500, n_jobs=8, random_state=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train model\n",
    "\n",
    "clf = LogisticRegression(random_state=0, n_jobs=8, max_iter=500)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fe25753",
   "metadata": {},
   "source": [
    "### Confusion mattrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "889d91dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Schema of confusion matrix**"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>True positives (TP)</td>\n",
       "      <td>False negatives (FN)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>False positives (FP)</td>\n",
       "      <td>True negatives (TN)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Ŷ = 1                 Ŷ = 0\n",
       "Y = 1   True positives (TP)  False negatives (FN)\n",
       "Y = 0  False positives (FP)   True negatives (TN)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Schema of confusion matrix\n",
    "\n",
    "df = pd.DataFrame([['True negatives (TN)', 'False positives (FP)'], ['False negatives (FN)', 'True positives (TP)']], index=['Y = 0', 'Y = 1'], columns=['Ŷ = 0', 'Ŷ = 1'])\n",
    "df = df.reindex(['Y = 1', 'Y = 0'])\n",
    "df = df[['Ŷ = 1', 'Ŷ = 0']]\n",
    "display(Markdown('**Schema of confusion matrix**'))\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "focal-strain",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function pretty_confusion_mattrix()\n",
    "\n",
    "def pretty_confusion_mattrix(y_label, y_pred, title=None):\n",
    "    \"\"\"Pretty print the confusion matrix computed by scikit-learn\"\"\"\n",
    "    _TN, _FP, _FN, _TP = confusion_matrix(y_label, y_pred).flatten()\n",
    "    array = [[_TP, _FN], [_FP, _TN]]\n",
    "    df = pd.DataFrame(array, index=['Y = 1', 'Y = 0'], columns=['Ŷ = 1', 'Ŷ = 0'])\n",
    "    if title is not None:\n",
    "        display(Markdown(title))\n",
    "    display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e51a8a6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Confusion matrix for the test dataset**"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>124</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>27</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1    124     17\n",
       "Y = 0     27     32"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# test dataset\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "pretty_confusion_mattrix(y_test, y_pred, title='**Confusion matrix for the test dataset**')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2b2f534c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function pretty_confusion_mattrix_by_subgroup()\n",
    "\n",
    "def pretty_confusion_mattrix_by_subgroup(X, col, X_test, y_label, y_pred, q=4):\n",
    "    \"\"\"Pretty print the confusion matrices by subgroup\n",
    "    \n",
    "    X: dataset\n",
    "    col: used for spliting in subgroups\n",
    "    X_test: test dataset\n",
    "    y_label: target for test dataset\n",
    "    y_pred: predictions for test dataset\n",
    "    q: quartile used for numerical column\"\"\"\n",
    "    \n",
    "    # if col is numeric, use quantile\n",
    "    cat = pd.qcut(X[col], q) if is_numeric_dtype(X[col]) else X[col]\n",
    "    # select test data\n",
    "    cat = cat.loc[X_test.index]\n",
    "    # switch y_pred to Series so as to be able to select by subgroup\n",
    "    y_pred = pd.Series(y_pred, index=y_label.index)\n",
    "    # loop on subgroups\n",
    "    for value in sorted(cat.unique()):\n",
    "        X_select = X_test.loc[cat == value]\n",
    "        pretty_confusion_mattrix(y_label.loc[X_select.index],\n",
    "                                 y_pred.loc[X_select.index],\n",
    "                                 title=f'**Subgroup**: {col} = {value}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e370b96e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: sex = female"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>35</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     35      4\n",
       "Y = 0      7     10"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: sex = male"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>89</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>20</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     89     13\n",
       "Y = 0     20     22"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pretty_confusion_mattrix_by_subgroup(X, 'sex', X_test, y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3ec6bb11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (18.999, 27.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>35</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     35      8\n",
       "Y = 0      3     14"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (27.0, 33.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     28      4\n",
       "Y = 0      7      7"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (33.0, 42.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>35</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     35      4\n",
       "Y = 0      6      5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (42.0, 75.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ŷ = 1</th>\n",
       "      <th>Ŷ = 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y = 1</th>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y = 0</th>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Ŷ = 1  Ŷ = 0\n",
       "Y = 1     26      1\n",
       "Y = 0     11      6"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pretty_confusion_mattrix_by_subgroup(X, 'Age in years', X_test, y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1b13d70",
   "metadata": {},
   "source": [
    "### Metrics derived from confusion matrix\n",
    "\n",
    "**Actual postitives**\n",
    "\n",
    "This number is the sum of the true positives and the false negatives, which can be viewed as missed\n",
    "true positives.\n",
    "\n",
    "$P = TP + FN$\n",
    "\n",
    "**Actual negatives**\n",
    "\n",
    "This number is the sum of the true negatives and the false positives, which again can be viewed as missed true negatives.\n",
    "\n",
    "$N = TN + FP$\n",
    "\n",
    "**Base rate**\n",
    "\n",
    "This number, sometimes also called the prevalence rate, represents the proportion of actual positives with respect to the entire data set.\n",
    "\n",
    "$BR = \\frac{P}{P + N}$\n",
    "\n",
    "**Positive rate**\n",
    "\n",
    "This number is the overall rate of positively classified instances, including both correct and incorrect decisions.\n",
    "\n",
    "$PR = \\frac{TP + FP}{P + N}$\n",
    "\n",
    "**Negative rate**\n",
    "\n",
    "This number is the ratio of negative classification, again irrespective of whether the decisions were correct or incorrect.\n",
    "\n",
    "$NR = \\frac{TN + FN}{P + N}$\n",
    "\n",
    "**Accuracy**\n",
    "\n",
    "This number is the ratio of the correctly classified instances (positive and negative) of all decisions.\n",
    "\n",
    "$ACC = \\frac{TP + TN}{P + N}$\n",
    "\n",
    "**Misclassiffication rate**\n",
    "\n",
    "This number is the ratio of the misclassified instances over all decisions.\n",
    "\n",
    "$MR = \\frac{FN + FP}{P + N}$\n",
    "\n",
    "**True positive rate (recall)**\n",
    "\n",
    "This number describes the proportions of correctly classified positive instances.\n",
    "\n",
    "$TPR = \\frac{TP}{P}$\n",
    "\n",
    "**True negative rate**\n",
    "\n",
    "This number describes the proportions of correctly classified negative instances.\n",
    "\n",
    "$TNR = \\frac{TN}{N}$\n",
    "\n",
    "**False positive rate**\n",
    "\n",
    "This number denotes the proportion of actual negatives which was falsely classified as positive.\n",
    "\n",
    "$FPR = \\frac{FP}{P}$\n",
    "\n",
    "**False negative rate (silence)**\n",
    "\n",
    "This number describes the proportion of actual positives which was misclassified as negative.\n",
    "\n",
    "$FNR = \\frac{FN}{N}$\n",
    "\n",
    "**False discovery rate (noise)**\n",
    "\n",
    "This number describes the share of misclassified positive classifications of all positive predictions.\n",
    "\n",
    "$FDR = \\frac{FP}{TP + FP}$\n",
    "\n",
    "**Positive predicted value (precision)**\n",
    "\n",
    "This number describes the ratio of samples which were correctly classified as positive from all the positive predictions.\n",
    "\n",
    "$PPV = \\frac{TP}{TP + FP}$\n",
    "\n",
    "**False omission rate**\n",
    "\n",
    "This number describes the proportion of false negative predictions of all negative predictions.\n",
    "\n",
    "$FOR = \\frac{FN}{TN + FN}$\n",
    "\n",
    "**Negative predicted value**\n",
    "\n",
    "This number describes the ratio of samples which were correctly classified as negative from all the negative predictions.\n",
    "\n",
    "$NPV = \\frac{TN}{TN + FN}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3fed5cec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function pretty_confusion_mattrix()\n",
    "\n",
    "def pretty_fairness_confusion_mattrix(y_label, y_pred, title=None):\n",
    "    \"\"\"Pretty print fairness confusion matrix\n",
    "    \n",
    "    y_label: target for test dataset\n",
    "    y_pred: predictions for test dataset\n",
    "    title: string to display in Markdown\"\"\"\n",
    "    \n",
    "    # compute fairness metrics\n",
    "    _TN, _FP, _FN, _TP = confusion_matrix(y_label, y_pred).flatten()\n",
    "    _P = _TP + _FN\n",
    "    _N = _FP + _TN\n",
    "    _BR = _P / (_P + _N)\n",
    "    _PR = (_TP + _FP) / (_P + _N)\n",
    "    _NR = (_TN + _FN) / (_P + _N)\n",
    "    _TPR = _TP / _P\n",
    "    _TNR = _TN / _N\n",
    "    _FDR = _FP / (_TP + _FP)\n",
    "    _FOR = _FN / (_TN + _FN)\n",
    "    \n",
    "    # build the output dataframe\n",
    "    array = [[_TP, _FN, f'TPR = {_TPR:.2f}'],\n",
    "             [_FP, _TN, f'TNR = {_TNR:.2f}'],\n",
    "             [f'FDR = {_FDR:.2f}', f'FOR = {_FOR:.2f}', f'BR = {_BR:.2f}'],\n",
    "             [f'PR = {_PR:.2f}', f'NR = {_NR:.2f}', ''],\n",
    "            ]\n",
    "    df = pd.DataFrame(array, index=['Y = 0', 'Y = 1', '', ' '], columns=['Ŷ = 0', 'Ŷ = 1', ''])\n",
    "    \n",
    "    \n",
    "    if title is not None:\n",
    "        display(Markdown(title))\n",
    "        \n",
    "    display(df.style.set_table_styles([{'selector': 'td', 'props':[('text-align', 'center')]},\n",
    "                                      {'selector': 'th', 'props': [('text-align', 'center')]}],\n",
    "                                      overwrite=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "412af2cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Fairness confusion matrix**"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_80f6c_ td {\n",
       "          text-align: center;\n",
       "    }    #T_80f6c_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_80f6c_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_80f6c_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_80f6c_row0_col0\" class=\"data row0 col0\" >124</td>\n",
       "                        <td id=\"T_80f6c_row0_col1\" class=\"data row0 col1\" >17</td>\n",
       "                        <td id=\"T_80f6c_row0_col2\" class=\"data row0 col2\" >TPR = 0.88</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_80f6c_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_80f6c_row1_col0\" class=\"data row1 col0\" >27</td>\n",
       "                        <td id=\"T_80f6c_row1_col1\" class=\"data row1 col1\" >32</td>\n",
       "                        <td id=\"T_80f6c_row1_col2\" class=\"data row1 col2\" >TNR = 0.54</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_80f6c_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_80f6c_row2_col0\" class=\"data row2 col0\" >FDR = 0.18</td>\n",
       "                        <td id=\"T_80f6c_row2_col1\" class=\"data row2 col1\" >FOR = 0.35</td>\n",
       "                        <td id=\"T_80f6c_row2_col2\" class=\"data row2 col2\" >BR = 0.70</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_80f6c_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_80f6c_row3_col0\" class=\"data row3 col0\" >PR = 0.76</td>\n",
       "                        <td id=\"T_80f6c_row3_col1\" class=\"data row3 col1\" >NR = 0.24</td>\n",
       "                        <td id=\"T_80f6c_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x18578910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pretty_fairness_confusion_mattrix(y_test, y_pred, title='**Fairness confusion matrix**')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "46626036",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function pretty_fairness_confusion_mattrix_by_subgroup()\n",
    "\n",
    "def pretty_fairness_confusion_mattrix_by_subgroup(X, col, X_test, y_label, y_pred, q=4):\n",
    "    \"\"\"Pretty print fairness confusion matrix by subgroup\n",
    "    \n",
    "    X: dataset\n",
    "    col: used for spliting in subgroups\n",
    "    X_test: test dataset\n",
    "    y_label: target for test dataset\n",
    "    y_pred: predictions for test dataset\n",
    "    q: quartile used for numerical colum\"\"\"\n",
    "    \n",
    "    # if col is numeric, use quantile\n",
    "    cat = pd.qcut(X[col], q) if is_numeric_dtype(X[col]) else X[col]\n",
    "    # select test data\n",
    "    cat = cat.loc[X_test.index]\n",
    "    # switch y_pred to Series so as to be able to select by subgroup\n",
    "    y_pred = pd.Series(y_pred, index=y_label.index)\n",
    "    # loop on subgroups\n",
    "    for value in sorted(cat.unique()):\n",
    "        X_select = X_test.loc[cat == value]\n",
    "        pretty_fairness_confusion_mattrix(y_label.loc[X_select.index],\n",
    "                                          y_pred.loc[X_select.index],\n",
    "                                          title=f'**Subgroup**: {col} = {value}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d5f25e90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: sex = female"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_b4639_ td {\n",
       "          text-align: center;\n",
       "    }    #T_b4639_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_b4639_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_b4639_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_b4639_row0_col0\" class=\"data row0 col0\" >35</td>\n",
       "                        <td id=\"T_b4639_row0_col1\" class=\"data row0 col1\" >4</td>\n",
       "                        <td id=\"T_b4639_row0_col2\" class=\"data row0 col2\" >TPR = 0.90</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b4639_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_b4639_row1_col0\" class=\"data row1 col0\" >7</td>\n",
       "                        <td id=\"T_b4639_row1_col1\" class=\"data row1 col1\" >10</td>\n",
       "                        <td id=\"T_b4639_row1_col2\" class=\"data row1 col2\" >TNR = 0.59</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b4639_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_b4639_row2_col0\" class=\"data row2 col0\" >FDR = 0.17</td>\n",
       "                        <td id=\"T_b4639_row2_col1\" class=\"data row2 col1\" >FOR = 0.29</td>\n",
       "                        <td id=\"T_b4639_row2_col2\" class=\"data row2 col2\" >BR = 0.70</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b4639_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_b4639_row3_col0\" class=\"data row3 col0\" >PR = 0.75</td>\n",
       "                        <td id=\"T_b4639_row3_col1\" class=\"data row3 col1\" >NR = 0.25</td>\n",
       "                        <td id=\"T_b4639_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x18519eb0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: sex = male"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_40203_ td {\n",
       "          text-align: center;\n",
       "    }    #T_40203_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_40203_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_40203_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_40203_row0_col0\" class=\"data row0 col0\" >89</td>\n",
       "                        <td id=\"T_40203_row0_col1\" class=\"data row0 col1\" >13</td>\n",
       "                        <td id=\"T_40203_row0_col2\" class=\"data row0 col2\" >TPR = 0.87</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_40203_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_40203_row1_col0\" class=\"data row1 col0\" >20</td>\n",
       "                        <td id=\"T_40203_row1_col1\" class=\"data row1 col1\" >22</td>\n",
       "                        <td id=\"T_40203_row1_col2\" class=\"data row1 col2\" >TNR = 0.52</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_40203_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_40203_row2_col0\" class=\"data row2 col0\" >FDR = 0.18</td>\n",
       "                        <td id=\"T_40203_row2_col1\" class=\"data row2 col1\" >FOR = 0.37</td>\n",
       "                        <td id=\"T_40203_row2_col2\" class=\"data row2 col2\" >BR = 0.71</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_40203_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_40203_row3_col0\" class=\"data row3 col0\" >PR = 0.76</td>\n",
       "                        <td id=\"T_40203_row3_col1\" class=\"data row3 col1\" >NR = 0.24</td>\n",
       "                        <td id=\"T_40203_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x17515fa0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pretty_fairness_confusion_mattrix_by_subgroup(X, 'sex', X_test, y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9819ed89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (18.999, 27.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_553dd_ td {\n",
       "          text-align: center;\n",
       "    }    #T_553dd_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_553dd_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_553dd_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_553dd_row0_col0\" class=\"data row0 col0\" >35</td>\n",
       "                        <td id=\"T_553dd_row0_col1\" class=\"data row0 col1\" >8</td>\n",
       "                        <td id=\"T_553dd_row0_col2\" class=\"data row0 col2\" >TPR = 0.81</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_553dd_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_553dd_row1_col0\" class=\"data row1 col0\" >3</td>\n",
       "                        <td id=\"T_553dd_row1_col1\" class=\"data row1 col1\" >14</td>\n",
       "                        <td id=\"T_553dd_row1_col2\" class=\"data row1 col2\" >TNR = 0.82</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_553dd_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_553dd_row2_col0\" class=\"data row2 col0\" >FDR = 0.08</td>\n",
       "                        <td id=\"T_553dd_row2_col1\" class=\"data row2 col1\" >FOR = 0.36</td>\n",
       "                        <td id=\"T_553dd_row2_col2\" class=\"data row2 col2\" >BR = 0.72</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_553dd_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_553dd_row3_col0\" class=\"data row3 col0\" >PR = 0.63</td>\n",
       "                        <td id=\"T_553dd_row3_col1\" class=\"data row3 col1\" >NR = 0.37</td>\n",
       "                        <td id=\"T_553dd_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1bdcfd60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (27.0, 33.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_3f76b_ td {\n",
       "          text-align: center;\n",
       "    }    #T_3f76b_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_3f76b_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_3f76b_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_3f76b_row0_col0\" class=\"data row0 col0\" >28</td>\n",
       "                        <td id=\"T_3f76b_row0_col1\" class=\"data row0 col1\" >4</td>\n",
       "                        <td id=\"T_3f76b_row0_col2\" class=\"data row0 col2\" >TPR = 0.88</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3f76b_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_3f76b_row1_col0\" class=\"data row1 col0\" >7</td>\n",
       "                        <td id=\"T_3f76b_row1_col1\" class=\"data row1 col1\" >7</td>\n",
       "                        <td id=\"T_3f76b_row1_col2\" class=\"data row1 col2\" >TNR = 0.50</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3f76b_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_3f76b_row2_col0\" class=\"data row2 col0\" >FDR = 0.20</td>\n",
       "                        <td id=\"T_3f76b_row2_col1\" class=\"data row2 col1\" >FOR = 0.36</td>\n",
       "                        <td id=\"T_3f76b_row2_col2\" class=\"data row2 col2\" >BR = 0.70</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3f76b_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_3f76b_row3_col0\" class=\"data row3 col0\" >PR = 0.76</td>\n",
       "                        <td id=\"T_3f76b_row3_col1\" class=\"data row3 col1\" >NR = 0.24</td>\n",
       "                        <td id=\"T_3f76b_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x174f21f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (33.0, 42.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_7e5b6_ td {\n",
       "          text-align: center;\n",
       "    }    #T_7e5b6_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_7e5b6_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_7e5b6_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_7e5b6_row0_col0\" class=\"data row0 col0\" >35</td>\n",
       "                        <td id=\"T_7e5b6_row0_col1\" class=\"data row0 col1\" >4</td>\n",
       "                        <td id=\"T_7e5b6_row0_col2\" class=\"data row0 col2\" >TPR = 0.90</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7e5b6_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_7e5b6_row1_col0\" class=\"data row1 col0\" >6</td>\n",
       "                        <td id=\"T_7e5b6_row1_col1\" class=\"data row1 col1\" >5</td>\n",
       "                        <td id=\"T_7e5b6_row1_col2\" class=\"data row1 col2\" >TNR = 0.45</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7e5b6_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_7e5b6_row2_col0\" class=\"data row2 col0\" >FDR = 0.15</td>\n",
       "                        <td id=\"T_7e5b6_row2_col1\" class=\"data row2 col1\" >FOR = 0.44</td>\n",
       "                        <td id=\"T_7e5b6_row2_col2\" class=\"data row2 col2\" >BR = 0.78</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7e5b6_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_7e5b6_row3_col0\" class=\"data row3 col0\" >PR = 0.82</td>\n",
       "                        <td id=\"T_7e5b6_row3_col1\" class=\"data row3 col1\" >NR = 0.18</td>\n",
       "                        <td id=\"T_7e5b6_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x4f7d4c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Subgroup**: Age in years = (42.0, 75.0]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_d2695_ td {\n",
       "          text-align: center;\n",
       "    }    #T_d2695_ th {\n",
       "          text-align: center;\n",
       "    }</style><table id=\"T_d2695_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Ŷ = 0</th>        <th class=\"col_heading level0 col1\" >Ŷ = 1</th>        <th class=\"col_heading level0 col2\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_d2695_level0_row0\" class=\"row_heading level0 row0\" >Y = 0</th>\n",
       "                        <td id=\"T_d2695_row0_col0\" class=\"data row0 col0\" >26</td>\n",
       "                        <td id=\"T_d2695_row0_col1\" class=\"data row0 col1\" >1</td>\n",
       "                        <td id=\"T_d2695_row0_col2\" class=\"data row0 col2\" >TPR = 0.96</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d2695_level0_row1\" class=\"row_heading level0 row1\" >Y = 1</th>\n",
       "                        <td id=\"T_d2695_row1_col0\" class=\"data row1 col0\" >11</td>\n",
       "                        <td id=\"T_d2695_row1_col1\" class=\"data row1 col1\" >6</td>\n",
       "                        <td id=\"T_d2695_row1_col2\" class=\"data row1 col2\" >TNR = 0.35</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d2695_level0_row2\" class=\"row_heading level0 row2\" ></th>\n",
       "                        <td id=\"T_d2695_row2_col0\" class=\"data row2 col0\" >FDR = 0.30</td>\n",
       "                        <td id=\"T_d2695_row2_col1\" class=\"data row2 col1\" >FOR = 0.14</td>\n",
       "                        <td id=\"T_d2695_row2_col2\" class=\"data row2 col2\" >BR = 0.61</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d2695_level0_row3\" class=\"row_heading level0 row3\" > </th>\n",
       "                        <td id=\"T_d2695_row3_col0\" class=\"data row3 col0\" >PR = 0.84</td>\n",
       "                        <td id=\"T_d2695_row3_col1\" class=\"data row3 col1\" >NR = 0.16</td>\n",
       "                        <td id=\"T_d2695_row3_col2\" class=\"data row3 col2\" ></td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1749f2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pretty_fairness_confusion_mattrix_by_subgroup(X, 'Age in years', X_test, y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}