Scikit-learn-prac.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Machine Learning\n",
    "\n",
    "This practical uses scikit-learn to make classification schemes for proteins based on metadata supplied in DUDE-E.features.10.tgz. The first part of the code just extracts files into the appropriate arrays.\n",
    "\n",
    "If you would like to explore other datasets, simply modify the combined_data function call in cell 4.\n",
    "\n",
    "Simon & Itai Dec 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'sklearn'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-cf3b031524a1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensemble\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDecisionTreeClassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDecisionTreeRegressor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear_model\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSGDClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sklearn'"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "import os\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "import numpy as np\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_curve, auc, confusion_matrix\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_dir():      \n",
    "    Active = []\n",
    "    Decoy = []\n",
    "    active_list=sorted(os.listdir(\"/home/sabs-r3/Documents/Data-science/ML/Active\"))\n",
    "    decoy_list = sorted(os.listdir(\"/home/sabs-r3/Documents/Data-science/ML/Decoy\"))\n",
    "    print(active_list)\n",
    "    for i in active_list:\n",
    "        Active.append(i)\n",
    "    for i in decoy_list:\n",
    "        Decoy.append(i)      \n",
    "    return Active, Decoy\n",
    "\n",
    "\n",
    "def combined_data(index):\n",
    "    if index >0 or index<9:\n",
    "        active_list = list_dir()[0]\n",
    "        os.chdir('/home/sabs-r3/Documents/Data-science/ML/Active')\n",
    "        active_data = pd.read_csv(active_list[index])\n",
    "        active_data[\"Activity\"] = [1]*len(active_data.index)\n",
    "        os.chdir('..')\n",
    "    \n",
    "        decoy_list = list_dir()[1]\n",
    "        os.chdir('/home/sabs-r3/Documents/Data-science/ML/Decoy')\n",
    "        decoy_data = pd.read_csv(decoy_list[index])\n",
    "        decoy_data[\"Activity\"] = [0]*len(decoy_data.index)\n",
    "        os.chdir('..')\n",
    "        data = pd.concat([active_data, decoy_data])\n",
    "    else:\n",
    "        print('Enter an index between 0 and 9')\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract data & split into training & testing sets\n",
    "data = combined_data(0) #modify this for other data!\n",
    "y = data['Activity'].to_numpy()\n",
    "X = data.to_numpy()[:, 1:-1]\n",
    "training_data, test_data, training_classes, test_classes = train_test_split(X, y, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The _predict_ function fits training data to a given classification scheme and immediately predicts classes in test data. This first one fits a random forest classification scheme. Comparison of the predicted to actual classes for the test data give a quick heuristic for function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(training_data, training_classes, test_data, clf):\n",
    "    clf.fit(training_data, training_classes)\n",
    "    predicted_classes = clf.predict(test_data)\n",
    "    return predicted_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_classes = predict(training_data, \n",
    "                       training_classes, \n",
    "                       test_data, \n",
    "                       RandomForestClassifier(random_state=0, n_estimators=500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simple test that it works\n",
    "matches = 0\n",
    "for n in range(len(pred_classes)):\n",
    "    if pred_classes[n] == test_classes[n]:\n",
    "        matches = matches + 1\n",
    "print(\"This matches %.2f percent of records\" % (matches*100 / len(pred_classes)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the prediction\n",
    "\n",
    "Receiver Operator Characteristic curves are a way of showing the true positive rate (tpr) versus the false positive rate (fpr) for a method. The area under this curve is a representation of how well the function works to minimise false positives and negatives.\n",
    "\n",
    "The confusion matrix is another way of visualising this: the diagonal top left to bottom right shows the numbers of points that are accurately classified (true positives and negatives). Conversely the opposite diagonal shows numbers of wrongly classified points."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_it(test_classes, pred_classes, title):\n",
    "    fpr, tpr, _ = roc_curve(test_classes, pred_classes)\n",
    "    plt.plot(fpr, tpr)\n",
    "    plt.plot([0,1],[0,1], 'o--')\n",
    "    plt.xlim([-0.01,1.0])\n",
    "    plt.ylim([0.0,1.0])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('ROC Curve for '+title)\n",
    "    plt.show()\n",
    "    print('Area Under Curve is %.3f' % auc(fpr, tpr))\n",
    "    print('Confusion matrix is:')\n",
    "    print(confusion_matrix(test_classes, pred_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_it(test_classes, pred_classes, \"Random Forest\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying other classification schemes\n",
    "\n",
    "The decision tree classifier is supposed to be a simpler version of the random forest (which takes multiple decision trees), but appears to work better on this data. We don't know why yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_classes = predict(training_data, \n",
    "                            training_classes, \n",
    "                            test_data, \n",
    "                            DecisionTreeClassifier(random_state=0))\n",
    "plot_it(test_classes, pred_classes, \"Decision Tree\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing with other datasets\n",
    "To prove that it's not this one dataset being strange, we tried it on others below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract data & split into training & testing sets\n",
    "data = combined_data(2) #modify this for other data!\n",
    "y = data['Activity'].to_numpy()\n",
    "X = data.to_numpy()[:, 1:-1]\n",
    "training_data, test_data, training_classes, test_classes = train_test_split(X, y, random_state=0)\n",
    "\n",
    "pred_classes = predict(training_data, \n",
    "                       training_classes, \n",
    "                       test_data, \n",
    "                       RandomForestClassifier(random_state=0, n_estimators=500))\n",
    "plot_it(test_classes, pred_classes, \"Random forest (dataset 2)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_classes = predict(training_data, \n",
    "                       training_classes, \n",
    "                       test_data, \n",
    "                       DecisionTreeClassifier(random_state=0))\n",
    "plot_it(test_classes, pred_classes, \"Decision tree (dataset 2)\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}