adams_sketchbook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ACQUIRE\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import unicodedata\n",
    "import re\n",
    "import nltk\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# PREPARE\n",
    "from nltk.tokenize.toktok import ToktokTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "import prepare\n",
    "from sklearn.model_selection import train_test_split\n",
    "# nltk.download('wordnet') - needed to run to download 'wordnet' resource to use lemmatize function\n",
    "\n",
    "# EXPLORE\n",
    "import matplotlib.pyplot as plt\n",
    "from math import sqrt\n",
    "from scipy import stats\n",
    "\n",
    "# MODELING\n",
    "from numpy.random import choice\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# EVALUATION\n",
    "from sklearn.metrics import classification_report, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>repo</th>\n",
       "      <th>language</th>\n",
       "      <th>readme_contents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gocodeup/codeup-setup-script</td>\n",
       "      <td>Shell</td>\n",
       "      <td># Codeup Setup Script\\n\\nSetup script for Code...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gocodeup/movies-application</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td># Movies Application\\n\\nFor this project, we w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>torvalds/linux</td>\n",
       "      <td>C</td>\n",
       "      <td>Linux kernel\\n============\\n\\nThere are severa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>beetbox/beets</td>\n",
       "      <td>Python</td>\n",
       "      <td>.. image:: https://img.shields.io/pypi/v/beets...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>scottschiller/SoundManager2</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td># SoundManager 2: JavaScript Sound for the Web...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>andyxialm/CutoLoadingView</td>\n",
       "      <td>Java</td>\n",
       "      <td>#CutoLoadingView\\n[![](https://jitpack.io/v/an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>rjsvieira/morphos</td>\n",
       "      <td>Java</td>\n",
       "      <td># Morphos\\n\\n![Current Version](https://img.sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>tuesda/SubmitDemo</td>\n",
       "      <td>Java</td>\n",
       "      <td>This is library project with a custom view tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>avenwu/support</td>\n",
       "      <td>Java</td>\n",
       "      <td>Support\\n========\\n\\nCustom Android support li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>cuub/sugared-list-animations</td>\n",
       "      <td>Java</td>\n",
       "      <td>SugaredListAnimations\\n=========\\n\\n&lt;br /&gt;Suga...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             repo    language  \\\n",
       "0    gocodeup/codeup-setup-script       Shell   \n",
       "1     gocodeup/movies-application  JavaScript   \n",
       "2                  torvalds/linux           C   \n",
       "3                   beetbox/beets      Python   \n",
       "4     scottschiller/SoundManager2  JavaScript   \n",
       "..                            ...         ...   \n",
       "197     andyxialm/CutoLoadingView        Java   \n",
       "198             rjsvieira/morphos        Java   \n",
       "199             tuesda/SubmitDemo        Java   \n",
       "200                avenwu/support        Java   \n",
       "201  cuub/sugared-list-animations        Java   \n",
       "\n",
       "                                       readme_contents  \n",
       "0    # Codeup Setup Script\\n\\nSetup script for Code...  \n",
       "1    # Movies Application\\n\\nFor this project, we w...  \n",
       "2    Linux kernel\\n============\\n\\nThere are severa...  \n",
       "3    .. image:: https://img.shields.io/pypi/v/beets...  \n",
       "4    # SoundManager 2: JavaScript Sound for the Web...  \n",
       "..                                                 ...  \n",
       "197  #CutoLoadingView\\n[![](https://jitpack.io/v/an...  \n",
       "198  # Morphos\\n\\n![Current Version](https://img.sh...  \n",
       "199  This is library project with a custom view tha...  \n",
       "200  Support\\n========\\n\\nCustom Android support li...  \n",
       "201  SugaredListAnimations\\n=========\\n\\n<br />Suga...  \n",
       "\n",
       "[202 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json('data.json')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = prepare.prep_data(df, 'readme_contents')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>repo</th>\n",
       "      <th>language</th>\n",
       "      <th>readme_contents</th>\n",
       "      <th>stemmed</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>clean</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>doc_length</th>\n",
       "      <th>words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gocodeup/movies-application</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td># Movies Application\\n\\nFor this project, we w...</td>\n",
       "      <td>movi applic for thi project we will be build a...</td>\n",
       "      <td>movie application for this project we will be ...</td>\n",
       "      <td>movie application building single page movie a...</td>\n",
       "      <td>386</td>\n",
       "      <td>406</td>\n",
       "      <td>[movie, application, building, single, page, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>torvalds/linux</td>\n",
       "      <td>C</td>\n",
       "      <td>Linux kernel\\n============\\n\\nThere are severa...</td>\n",
       "      <td>linux kernel there are sever guid for kernel d...</td>\n",
       "      <td>linux kernel there are several guide for kerne...</td>\n",
       "      <td>linux kernel several guide kernel developer us...</td>\n",
       "      <td>45</td>\n",
       "      <td>65</td>\n",
       "      <td>[linux, kernel, several, guide, kernel, develo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>beetbox/beets</td>\n",
       "      <td>Python</td>\n",
       "      <td>.. image:: https://img.shields.io/pypi/v/beets...</td>\n",
       "      <td>imag http img shield io pypi v beet svg target...</td>\n",
       "      <td>image http img shield io pypi v beet svg targe...</td>\n",
       "      <td>image img shield io pypi v beet svg target pyp...</td>\n",
       "      <td>274</td>\n",
       "      <td>455</td>\n",
       "      <td>[image, img, shield, io, pypi, v, beet, svg, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>scottschiller/SoundManager2</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td># SoundManager 2: JavaScript Sound for the Web...</td>\n",
       "      <td>soundmanag javascript sound for the web by wra...</td>\n",
       "      <td>soundmanager javascript sound for the web by w...</td>\n",
       "      <td>soundmanager javascript sound web wrapping ext...</td>\n",
       "      <td>339</td>\n",
       "      <td>615</td>\n",
       "      <td>[soundmanager, javascript, sound, web, wrappin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CreateJS/SoundJS</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td># SoundJS\\r\\n\\r\\nSoundJS is a library to make ...</td>\n",
       "      <td>soundj soundj is a librari to make work with a...</td>\n",
       "      <td>soundjs soundjs is a library to make working w...</td>\n",
       "      <td>soundjs soundjs make working audio web easier ...</td>\n",
       "      <td>198</td>\n",
       "      <td>314</td>\n",
       "      <td>[soundjs, soundjs, make, working, audio, web, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>andyxialm/CutoLoadingView</td>\n",
       "      <td>Java</td>\n",
       "      <td>#CutoLoadingView\\n[![](https://jitpack.io/v/an...</td>\n",
       "      <td>cutoloadingview http jitpack io v andyxialm cu...</td>\n",
       "      <td>cutoloadingview http jitpack io v andyxialm cu...</td>\n",
       "      <td>cutoloadingview jitpack io v andyxialm cutoloa...</td>\n",
       "      <td>78</td>\n",
       "      <td>132</td>\n",
       "      <td>[cutoloadingview, jitpack, io, v, andyxialm, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>rjsvieira/morphos</td>\n",
       "      <td>Java</td>\n",
       "      <td># Morphos\\n\\n![Current Version](https://img.sh...</td>\n",
       "      <td>morpho current version http img shield io badg...</td>\n",
       "      <td>morphos current version http img shield io bad...</td>\n",
       "      <td>morphos current version img shield io badge cu...</td>\n",
       "      <td>209</td>\n",
       "      <td>674</td>\n",
       "      <td>[morphos, current, version, img, shield, io, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>tuesda/SubmitDemo</td>\n",
       "      <td>Java</td>\n",
       "      <td>This is library project with a custom view tha...</td>\n",
       "      <td>thi is librari project with a custom view that...</td>\n",
       "      <td>this is library project with a custom view tha...</td>\n",
       "      <td>custom view implement concept submit button dr...</td>\n",
       "      <td>17</td>\n",
       "      <td>92</td>\n",
       "      <td>[custom, view, implement, concept, submit, but...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>avenwu/support</td>\n",
       "      <td>Java</td>\n",
       "      <td>Support\\n========\\n\\nCustom Android support li...</td>\n",
       "      <td>support custom android support librari includ ...</td>\n",
       "      <td>support custom android support library include...</td>\n",
       "      <td>support custom android support include useful ...</td>\n",
       "      <td>97</td>\n",
       "      <td>261</td>\n",
       "      <td>[support, custom, android, support, include, u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>cuub/sugared-list-animations</td>\n",
       "      <td>Java</td>\n",
       "      <td>SugaredListAnimations\\n=========\\n\\n&lt;br /&gt;Suga...</td>\n",
       "      <td>sugaredlistanim br sugaredlistanim is a librar...</td>\n",
       "      <td>sugaredlistanimations br sugaredlistanimations...</td>\n",
       "      <td>sugaredlistanimations br sugaredlistanimations...</td>\n",
       "      <td>23</td>\n",
       "      <td>39</td>\n",
       "      <td>[sugaredlistanimations, br, sugaredlistanimati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>185 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             repo    language  \\\n",
       "0     gocodeup/movies-application  JavaScript   \n",
       "1                  torvalds/linux           C   \n",
       "2                   beetbox/beets      Python   \n",
       "3     scottschiller/SoundManager2  JavaScript   \n",
       "4                CreateJS/SoundJS  JavaScript   \n",
       "..                            ...         ...   \n",
       "180     andyxialm/CutoLoadingView        Java   \n",
       "181             rjsvieira/morphos        Java   \n",
       "182             tuesda/SubmitDemo        Java   \n",
       "183                avenwu/support        Java   \n",
       "184  cuub/sugared-list-animations        Java   \n",
       "\n",
       "                                       readme_contents  \\\n",
       "0    # Movies Application\\n\\nFor this project, we w...   \n",
       "1    Linux kernel\\n============\\n\\nThere are severa...   \n",
       "2    .. image:: https://img.shields.io/pypi/v/beets...   \n",
       "3    # SoundManager 2: JavaScript Sound for the Web...   \n",
       "4    # SoundJS\\r\\n\\r\\nSoundJS is a library to make ...   \n",
       "..                                                 ...   \n",
       "180  #CutoLoadingView\\n[![](https://jitpack.io/v/an...   \n",
       "181  # Morphos\\n\\n![Current Version](https://img.sh...   \n",
       "182  This is library project with a custom view tha...   \n",
       "183  Support\\n========\\n\\nCustom Android support li...   \n",
       "184  SugaredListAnimations\\n=========\\n\\n<br />Suga...   \n",
       "\n",
       "                                               stemmed  \\\n",
       "0    movi applic for thi project we will be build a...   \n",
       "1    linux kernel there are sever guid for kernel d...   \n",
       "2    imag http img shield io pypi v beet svg target...   \n",
       "3    soundmanag javascript sound for the web by wra...   \n",
       "4    soundj soundj is a librari to make work with a...   \n",
       "..                                                 ...   \n",
       "180  cutoloadingview http jitpack io v andyxialm cu...   \n",
       "181  morpho current version http img shield io badg...   \n",
       "182  thi is librari project with a custom view that...   \n",
       "183  support custom android support librari includ ...   \n",
       "184  sugaredlistanim br sugaredlistanim is a librar...   \n",
       "\n",
       "                                            lemmatized  \\\n",
       "0    movie application for this project we will be ...   \n",
       "1    linux kernel there are several guide for kerne...   \n",
       "2    image http img shield io pypi v beet svg targe...   \n",
       "3    soundmanager javascript sound for the web by w...   \n",
       "4    soundjs soundjs is a library to make working w...   \n",
       "..                                                 ...   \n",
       "180  cutoloadingview http jitpack io v andyxialm cu...   \n",
       "181  morphos current version http img shield io bad...   \n",
       "182  this is library project with a custom view tha...   \n",
       "183  support custom android support library include...   \n",
       "184  sugaredlistanimations br sugaredlistanimations...   \n",
       "\n",
       "                                                 clean  stopwords_removed  \\\n",
       "0    movie application building single page movie a...                386   \n",
       "1    linux kernel several guide kernel developer us...                 45   \n",
       "2    image img shield io pypi v beet svg target pyp...                274   \n",
       "3    soundmanager javascript sound web wrapping ext...                339   \n",
       "4    soundjs soundjs make working audio web easier ...                198   \n",
       "..                                                 ...                ...   \n",
       "180  cutoloadingview jitpack io v andyxialm cutoloa...                 78   \n",
       "181  morphos current version img shield io badge cu...                209   \n",
       "182  custom view implement concept submit button dr...                 17   \n",
       "183  support custom android support include useful ...                 97   \n",
       "184  sugaredlistanimations br sugaredlistanimations...                 23   \n",
       "\n",
       "     doc_length                                              words  \n",
       "0           406  [movie, application, building, single, page, m...  \n",
       "1            65  [linux, kernel, several, guide, kernel, develo...  \n",
       "2           455  [image, img, shield, io, pypi, v, beet, svg, t...  \n",
       "3           615  [soundmanager, javascript, sound, web, wrappin...  \n",
       "4           314  [soundjs, soundjs, make, working, audio, web, ...  \n",
       "..          ...                                                ...  \n",
       "180         132  [cutoloadingview, jitpack, io, v, andyxialm, c...  \n",
       "181         674  [morphos, current, version, img, shield, io, b...  \n",
       "182          92  [custom, view, implement, concept, submit, but...  \n",
       "183         261  [support, custom, android, support, include, u...  \n",
       "184          39  [sugaredlistanimations, br, sugaredlistanimati...  \n",
       "\n",
       "[185 rows x 9 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HTML                30\n",
       "Ruby                28\n",
       "JavaScript          28\n",
       "C++                 28\n",
       "Python              28\n",
       "Java                24\n",
       "Jupyter Notebook     6\n",
       "CSS                  4\n",
       "TypeScript           3\n",
       "C                    2\n",
       "C#                   2\n",
       "Scala                2\n",
       "Name: language, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 185 entries, 0 to 184\n",
      "Data columns (total 9 columns):\n",
      " #   Column             Non-Null Count  Dtype \n",
      "---  ------             --------------  ----- \n",
      " 0   repo               185 non-null    object\n",
      " 1   language           185 non-null    object\n",
      " 2   readme_contents    185 non-null    object\n",
      " 3   stemmed            185 non-null    object\n",
      " 4   lemmatized         185 non-null    object\n",
      " 5   clean              185 non-null    object\n",
      " 6   stopwords_removed  185 non-null    int64 \n",
      " 7   doc_length         185 non-null    int64 \n",
      " 8   words              185 non-null    object\n",
      "dtypes: int64(2), object(7)\n",
      "memory usage: 13.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_validate, test = train_test_split(df, stratify = df.language, test_size = .15, random_state= 123)\n",
    "train, validate = train_test_split(train_validate, stratify = train_validate.language, test_size = .20, random_state = 123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((125, 9), (32, 9), (28, 9))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, validate.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HTML                20\n",
       "C++                 19\n",
       "Python              19\n",
       "Ruby                19\n",
       "JavaScript          19\n",
       "Java                16\n",
       "Jupyter Notebook     4\n",
       "C#                   2\n",
       "TypeScript           2\n",
       "CSS                  2\n",
       "C                    2\n",
       "Scala                1\n",
       "Name: language, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 9)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HTML                5\n",
       "C++                 5\n",
       "Ruby                5\n",
       "JavaScript          5\n",
       "Python              5\n",
       "Java                4\n",
       "CSS                 1\n",
       "Jupyter Notebook    1\n",
       "Scala               1\n",
       "Name: language, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate.language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create series objects for each top_code_clean that is a string of words joined on spaces to make it 1 continious string  \n",
    "javascript_words = ' '.join(train[train.language=='JavaScript'].clean)\n",
    "python_words = ' '.join(train[train.language=='Python'].clean)\n",
    "ruby_words = ' '.join(train[train.language=='Ruby'].clean)\n",
    "html_words = ' '.join(train[train.language=='HTML'].clean)\n",
    "c_plus_plus_words = ' '.join(train[train.language=='C++'].clean)\n",
    "all_words = ' '.join(train.clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploration\n",
    "\n",
    "Explore the data that you have scraped. Here are some ideas for exploration:\n",
    "\n",
    "- What are the most common words in READMEs?\n",
    "- What does the distribution of IDFs look like for the most common words?\n",
    "- Does the length of the README vary by programming language?\n",
    "- Do different programming languages use a different number of unique words?\n",
    "\n",
    "### What are the most common words in READMEs?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words_df = pd.DataFrame(all_words.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning      545\n",
       "data          506\n",
       "machine       462\n",
       "python        345\n",
       "html          307\n",
       "code          268\n",
       "build         252\n",
       "tensorflow    248\n",
       "io            239\n",
       "j             232\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_words_df.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What does the distribution of IDFs look like for the most common words?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def idf(words_df):\n",
    "    '''\n",
    "    Takes in a dataframe made of a single column where each row is a single word and \n",
    "    returns a dataframe containing the top 10 most frequent words,\n",
    "    the number of documents each word shows up in, and the idf value for each word\n",
    "    '''\n",
    "    \n",
    "    # Creates a dictionary of the top ten words and their number of counts\n",
    "    top_10_words = words_df[0].value_counts(dropna=False).head(10).keys().tolist()\n",
    "    top_10_counts = words_df[0].value_counts(dropna=False).head(10).tolist()\n",
    "    word_counts = dict(zip(top_10_words, top_10_counts))\n",
    "    \n",
    "    # Derives the 10 most common words from the given dataframe of words\n",
    "    most_common_words = [word[0] for word in list(words_df.value_counts().head(10).index)]\n",
    "    \n",
    "    # This dictionary will store how many documents each word appears in \n",
    "    appearances_dict = dict.fromkeys(most_common_words)\n",
    "    \n",
    "    # The total number of documents is based on the number of rows in the train dataframe\n",
    "    number_of_documents = train.shape[0]\n",
    "    \n",
    "    # This is essentially a list containing the contents of the words column in the dataframe. It is a list of lists.\n",
    "    list_of_wordlists = list(train.words.values)\n",
    "    \n",
    "    # Start iterating through the list of common words. We want to collect information for each one.\n",
    "    for word in most_common_words:\n",
    "        \n",
    "        # Set the initial number of documents that the word appears in to zero\n",
    "        number_of_appearances = 0\n",
    "        \n",
    "        # Start iterating through the list made from the words column in the dataframe \n",
    "        for words in list_of_wordlists:\n",
    "            \n",
    "            # If the current word is in a document, add 1 to the number of appearances and then move to the next document\n",
    "            if word in words:\n",
    "                number_of_appearances += 1\n",
    "        \n",
    "        # Once all the documents have been iterated through, add the sum total of all appearances to our appearances dictionary\n",
    "        appearances_dict[word] = number_of_appearances\n",
    "        \n",
    "    # Create a new dictionary that will contain the IDF values for each word\n",
    "    idf_dict = dict.fromkeys(most_common_words)\n",
    "    \n",
    "    # Start iterating through the list of common words again, using the number of appearances and the total number of documents to calculate the IDF and update the relevant key:value in the dictionary\n",
    "    for word in most_common_words:\n",
    "        idf_dict[word] = np.log(number_of_documents / appearances_dict[word])\n",
    "    \n",
    "    # Create dataframes containing the information of the total counts, the number of documents, and the idf value\n",
    "    idf_df1 = pd.DataFrame.from_dict(word_counts, orient='index', columns=['total_count']).reset_index().rename(columns={'index':'word'})\n",
    "    idf_df2 = pd.DataFrame.from_dict(appearances_dict, orient='index', columns=['num_of_documents']).reset_index().rename(columns={'index':'word'})\n",
    "    idf_df3 = pd.DataFrame.from_dict(idf_dict, orient='index', columns=['idf_value']).reset_index().rename(columns={'index':'word'})\n",
    "    \n",
    "    # Merge the dataframes into a single dataframe\n",
    "    idf_df = pd.merge(idf_df1, idf_df2, left_on = 'word', right_on = 'word')\n",
    "    idf_df = pd.merge(idf_df, idf_df3, left_on = 'word', right_on = 'word')\n",
    "    \n",
    "    # Return the dataframe\n",
    "    return idf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>total_count</th>\n",
       "      <th>num_of_documents</th>\n",
       "      <th>idf_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>learning</td>\n",
       "      <td>545</td>\n",
       "      <td>15</td>\n",
       "      <td>2.120264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data</td>\n",
       "      <td>506</td>\n",
       "      <td>38</td>\n",
       "      <td>1.190728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>machine</td>\n",
       "      <td>462</td>\n",
       "      <td>20</td>\n",
       "      <td>1.832581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>python</td>\n",
       "      <td>345</td>\n",
       "      <td>28</td>\n",
       "      <td>1.496109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>html</td>\n",
       "      <td>307</td>\n",
       "      <td>52</td>\n",
       "      <td>0.877070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>code</td>\n",
       "      <td>268</td>\n",
       "      <td>66</td>\n",
       "      <td>0.638659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>build</td>\n",
       "      <td>252</td>\n",
       "      <td>52</td>\n",
       "      <td>0.877070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>tensorflow</td>\n",
       "      <td>248</td>\n",
       "      <td>5</td>\n",
       "      <td>3.218876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>io</td>\n",
       "      <td>239</td>\n",
       "      <td>53</td>\n",
       "      <td>0.858022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>j</td>\n",
       "      <td>232</td>\n",
       "      <td>26</td>\n",
       "      <td>1.570217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         word  total_count  num_of_documents  idf_value\n",
       "0    learning          545                15   2.120264\n",
       "1        data          506                38   1.190728\n",
       "2     machine          462                20   1.832581\n",
       "3      python          345                28   1.496109\n",
       "4        html          307                52   0.877070\n",
       "5        code          268                66   0.638659\n",
       "6       build          252                52   0.877070\n",
       "7  tensorflow          248                 5   3.218876\n",
       "8          io          239                53   0.858022\n",
       "9           j          232                26   1.570217"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf(all_words_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Does the length of the README vary by programming language?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "language\n",
       "TypeScript          2227.500000\n",
       "Python              1585.368421\n",
       "Ruby                 450.578947\n",
       "C++                  396.894737\n",
       "C#                   395.000000\n",
       "JavaScript           343.789474\n",
       "Java                 286.750000\n",
       "Scala                283.000000\n",
       "HTML                 265.800000\n",
       "C                    210.000000\n",
       "CSS                  170.000000\n",
       "Jupyter Notebook     106.750000\n",
       "Name: doc_length, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('language').doc_length.mean().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def t_test_loop(target, alpha = 0.05):    \n",
    "    '''\n",
    "    This function is dependent on a globally defined dataframe named 'train' that contains a 'language' column containing strings\n",
    "    The target is the header for the column being investigated. The column being investigated must be a numeric type.\n",
    "    Comparative t-tests are run on the target column between every possible pair of languages represented in the train dataset.\n",
    "    The function prints out the result of every t-test that has a p-value below alpha. \n",
    "    '''\n",
    "    # Start by developing two identical lists of all of the languages represented in the train dataset\n",
    "    # We will use the series above to form our list so our output is arranged in a similar order\n",
    "    # When this is converted to a function, we can generate the list in a more generalized manner\n",
    "    train_language_list_1 = list(train.groupby('language')[target].mean().sort_values(ascending=False).index)\n",
    "    train_language_list_2 = list(train.groupby('language')[target].mean().sort_values(ascending=False).index)\n",
    "\n",
    "    # This empty list will hold information about which pairs have been tested. \n",
    "    # If the python and javascript pair has already be tested, then we do not need to test the javascript and python pair\n",
    "    testing_pairs = []\n",
    "\n",
    "    for language_1 in train_language_list_1: # Iterates through list 1\n",
    "        for language_2 in train_language_list_2: # Iterates through list 2 in entirety for each element in list 1\n",
    "\n",
    "            if language_1 == language_2: # Cannot run a t-test against itself, so skip the test if the two list elements are identical\n",
    "                continue\n",
    "\n",
    "            else:\n",
    "                alpha = 0.05 # Set alpha\n",
    "\n",
    "                # Run the t-test and store the t-statistic and the p-value\n",
    "                stat, p = stats.ttest_ind(train[train.language == language_1][target], train[train.language == language_2][target])\n",
    "\n",
    "                # If the p-value is statistically significant we print the results, otherwise we do nothing\n",
    "                if p/2 < alpha:\n",
    "\n",
    "                    # Creating strings to represent the pair that is being tested (eg. 'Python and JavaScript' & 'JavaScript and Python')\n",
    "                    testing_pair_1 = language_1 + \" \" + language_2\n",
    "                    testing_pair_2 = language_2 + \" \" + language_1\n",
    "\n",
    "                    # If this unique pair has not yet been tested:\n",
    "                    if (testing_pair_1 not in testing_pairs) and (testing_pair_2 not in testing_pairs):\n",
    "\n",
    "                        # Add this pair to the testing_pairs list so that we do not output duplicate t-test results\n",
    "                        testing_pairs.append(testing_pair_1)\n",
    "                        testing_pairs.append(testing_pair_2)\n",
    "\n",
    "                        # Print the results of the test\n",
    "                        print(\"----------------\")\n",
    "                        print(f\"{target} T-Test: {language_1} & {language_2}\")\n",
    "                        print(\"----------------\")\n",
    "                        print(\"Hypotheses:\")\n",
    "                        print(f\"H_0: There is no difference in the mean {target} of {language_1} and {language_2}\")\n",
    "                        print(f\"H_a: There is a difference in the mean {target} of {language_1} and {language_2}\")\n",
    "                        print('\\n')\n",
    "                        print(f\"p-value: {p/2}\")\n",
    "                        print(f\"t-statistic: {stat}\")\n",
    "                        print(f\"We reject the null hypothesis\")\n",
    "                        print(\"\\n\")\n",
    "                        if stat < 0:\n",
    "                            print(f\"The mean readme {target} for {language_1} is smaller than {language_2}\")\n",
    "                        elif stat > 0:\n",
    "                            print(f\"The mean readme {target} for {language_1} is larger than {language_2}\")\n",
    "                        print('\\n','\\n')\n",
    "\n",
    "                        # If the pair had already been tested, do not print any results and continue through the loop\n",
    "                    else:\n",
    "                        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------\n",
      "doc_length T-Test: TypeScript & Ruby\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of TypeScript and Ruby\n",
      "H_a: There is a difference in the mean doc_length of TypeScript and Ruby\n",
      "\n",
      "\n",
      "p-value: 0.0018824887340140337\n",
      "t-statistic: 3.3000440250787317\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for TypeScript is larger than Ruby\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "doc_length T-Test: TypeScript & C++\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of TypeScript and C++\n",
      "H_a: There is a difference in the mean doc_length of TypeScript and C++\n",
      "\n",
      "\n",
      "p-value: 0.001967416577897316\n",
      "t-statistic: 3.2804469201019173\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for TypeScript is larger than C++\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "doc_length T-Test: TypeScript & JavaScript\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of TypeScript and JavaScript\n",
      "H_a: There is a difference in the mean doc_length of TypeScript and JavaScript\n",
      "\n",
      "\n",
      "p-value: 0.0007953245370995328\n",
      "t-statistic: 3.679992479036958\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for TypeScript is larger than JavaScript\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "doc_length T-Test: TypeScript & Java\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of TypeScript and Java\n",
      "H_a: There is a difference in the mean doc_length of TypeScript and Java\n",
      "\n",
      "\n",
      "p-value: 0.0014542199319847523\n",
      "t-statistic: 3.508886951190981\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for TypeScript is larger than Java\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "doc_length T-Test: TypeScript & HTML\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of TypeScript and HTML\n",
      "H_a: There is a difference in the mean doc_length of TypeScript and HTML\n",
      "\n",
      "\n",
      "p-value: 0.0028359379028360387\n",
      "t-statistic: 3.097895663111655\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for TypeScript is larger than HTML\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "doc_length T-Test: Ruby & Jupyter Notebook\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean doc_length of Ruby and Jupyter Notebook\n",
      "H_a: There is a difference in the mean doc_length of Ruby and Jupyter Notebook\n",
      "\n",
      "\n",
      "p-value: 0.03884654106393356\n",
      "t-statistic: 1.8549828129654342\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme doc_length for Ruby is larger than Jupyter Notebook\n",
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "t_test_loop('doc_length')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do different programming languages use a different number of unique words?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_word_count = {}\n",
    "\n",
    "train_language_list = list(train.groupby('language')['doc_length'].mean().sort_values(ascending=False).index)\n",
    "\n",
    "for language in train_language_list:\n",
    "    unique_word_count[language] = len(set(' '.join(train[train.language==language].clean).split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_word_count = pd.DataFrame.from_dict(unique_word_count, orient='index').rename(columns={0: 'num_unique_words'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_unique_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Python</th>\n",
       "      <td>6144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ruby</th>\n",
       "      <td>2257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C++</th>\n",
       "      <td>2109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JavaScript</th>\n",
       "      <td>1973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HTML</th>\n",
       "      <td>1656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TypeScript</th>\n",
       "      <td>1289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Java</th>\n",
       "      <td>1278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C#</th>\n",
       "      <td>339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jupyter Notebook</th>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CSS</th>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Scala</th>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  num_unique_words\n",
       "Python                        6144\n",
       "Ruby                          2257\n",
       "C++                           2109\n",
       "JavaScript                    1973\n",
       "HTML                          1656\n",
       "TypeScript                    1289\n",
       "Java                          1278\n",
       "C#                             339\n",
       "Jupyter Notebook               267\n",
       "C                              223\n",
       "CSS                            199\n",
       "Scala                          151"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_word_count.sort_values(by='num_unique_words', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While this gives us a total count of the number of unique words each programming language has connected to it, this is likely to be correlated both to the total document length AND the number of observations that each language has. Languages with more observations likely have more opportunities to expand the unique word list. \n",
    "\n",
    "A better metric would be the average number of unique words that each language has. To do this we will need to add a column to our dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['unique_word_count'] = train.words.apply(lambda x: len(set(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>repo</th>\n",
       "      <th>language</th>\n",
       "      <th>readme_contents</th>\n",
       "      <th>stemmed</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>clean</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>doc_length</th>\n",
       "      <th>words</th>\n",
       "      <th>unique_word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>censusreporter/censusreporter</td>\n",
       "      <td>HTML</td>\n",
       "      <td>About Census Reporter\\n=====================\\n...</td>\n",
       "      <td>about censu report the unit state censu bureau...</td>\n",
       "      <td>about census reporter the united state census ...</td>\n",
       "      <td>census reporter united state census bureau pro...</td>\n",
       "      <td>1573</td>\n",
       "      <td>2789</td>\n",
       "      <td>[census, reporter, united, state, census, bure...</td>\n",
       "      <td>861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>StarterSquad/ngseed</td>\n",
       "      <td>HTML</td>\n",
       "      <td>ngseed\\n======\\n\\nAngularJS/RequireJS seed pro...</td>\n",
       "      <td>ngseed angularj requirej seed project document...</td>\n",
       "      <td>ngseed angularjs requirejs seed project docume...</td>\n",
       "      <td>ngseed angularjs requirejs seed documentation ...</td>\n",
       "      <td>115</td>\n",
       "      <td>265</td>\n",
       "      <td>[ngseed, angularjs, requirejs, seed, documenta...</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>igrigorik/decisiontree</td>\n",
       "      <td>Ruby</td>\n",
       "      <td># Decision Tree\\n\\nA Ruby library which implem...</td>\n",
       "      <td>decis tree a rubi librari which implement id3 ...</td>\n",
       "      <td>decision tree a ruby library which implement i...</td>\n",
       "      <td>decision tree ruby implement id3 information g...</td>\n",
       "      <td>101</td>\n",
       "      <td>264</td>\n",
       "      <td>[decision, tree, ruby, implement, id3, informa...</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>tensorflow/tensorflow</td>\n",
       "      <td>C++</td>\n",
       "      <td>&lt;div align=\"center\"&gt;\\n  &lt;img src=\"https://www....</td>\n",
       "      <td>div align center img src http www tensorflow o...</td>\n",
       "      <td>div align center img src http www tensorflow o...</td>\n",
       "      <td>div align center img src tensorflow image tf l...</td>\n",
       "      <td>513</td>\n",
       "      <td>1586</td>\n",
       "      <td>[div, align, center, img, src, tensorflow, ima...</td>\n",
       "      <td>324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>torvalds/linux</td>\n",
       "      <td>C</td>\n",
       "      <td>Linux kernel\\n============\\n\\nThere are severa...</td>\n",
       "      <td>linux kernel there are sever guid for kernel d...</td>\n",
       "      <td>linux kernel there are several guide for kerne...</td>\n",
       "      <td>linux kernel several guide kernel developer us...</td>\n",
       "      <td>45</td>\n",
       "      <td>65</td>\n",
       "      <td>[linux, kernel, several, guide, kernel, develo...</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>designmodo/Flat-UI</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>\\n# [Flat UI Free](https://designmodo.github.i...</td>\n",
       "      <td>flat ui free http designmodo github io flat ui...</td>\n",
       "      <td>flat ui free http designmodo github io flat ui...</td>\n",
       "      <td>flat ui free designmodo io flat ui flat ui bea...</td>\n",
       "      <td>219</td>\n",
       "      <td>392</td>\n",
       "      <td>[flat, ui, free, designmodo, io, flat, ui, fla...</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>musescore/MuseScore</td>\n",
       "      <td>C++</td>\n",
       "      <td>![MuseScore](mscore/data/musescore_logo_full.p...</td>\n",
       "      <td>musescor mscore data musescor logo full png mu...</td>\n",
       "      <td>musescore mscore data musescore logo full png ...</td>\n",
       "      <td>musescore mscore data musescore logo full png ...</td>\n",
       "      <td>279</td>\n",
       "      <td>583</td>\n",
       "      <td>[musescore, mscore, data, musescore, logo, ful...</td>\n",
       "      <td>311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Hextris/hextris</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Hextris\\n==========\\n\\n&lt;img src=\"images/twitte...</td>\n",
       "      <td>hextri img src imag twitter opengraph png widt...</td>\n",
       "      <td>hextris img src image twitter opengraph png wi...</td>\n",
       "      <td>hextris img src image twitter opengraph png wi...</td>\n",
       "      <td>117</td>\n",
       "      <td>148</td>\n",
       "      <td>[hextris, img, src, image, twitter, opengraph,...</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>patriciogonzalezvivo/KinectCoreVision</td>\n",
       "      <td>C++</td>\n",
       "      <td>&lt;iframe class=\"roundPhoto\" src=\"http://player....</td>\n",
       "      <td>ifram class roundphoto src http player vimeo c...</td>\n",
       "      <td>iframe class roundphoto src http player vimeo ...</td>\n",
       "      <td>iframe class roundphoto src player vimeo video...</td>\n",
       "      <td>57</td>\n",
       "      <td>127</td>\n",
       "      <td>[iframe, class, roundphoto, src, player, vimeo...</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>jvalen/pixel-art-react</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>&lt;p align=\"center\"&gt;\\n  &lt;img width=\"200\" src=\"sc...</td>\n",
       "      <td>p align center img width 200 src screenshot tr...</td>\n",
       "      <td>p align center img width 200 src screenshots t...</td>\n",
       "      <td>p align center img width 200 src screenshots t...</td>\n",
       "      <td>188</td>\n",
       "      <td>361</td>\n",
       "      <td>[p, align, center, img, width, 200, src, scree...</td>\n",
       "      <td>214</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      repo    language  \\\n",
       "26           censusreporter/censusreporter        HTML   \n",
       "135                    StarterSquad/ngseed        HTML   \n",
       "77                  igrigorik/decisiontree        Ruby   \n",
       "84                   tensorflow/tensorflow         C++   \n",
       "1                           torvalds/linux           C   \n",
       "..                                     ...         ...   \n",
       "16                      designmodo/Flat-UI  JavaScript   \n",
       "5                      musescore/MuseScore         C++   \n",
       "48                         Hextris/hextris  JavaScript   \n",
       "147  patriciogonzalezvivo/KinectCoreVision         C++   \n",
       "36                  jvalen/pixel-art-react  JavaScript   \n",
       "\n",
       "                                       readme_contents  \\\n",
       "26   About Census Reporter\\n=====================\\n...   \n",
       "135  ngseed\\n======\\n\\nAngularJS/RequireJS seed pro...   \n",
       "77   # Decision Tree\\n\\nA Ruby library which implem...   \n",
       "84   <div align=\"center\">\\n  <img src=\"https://www....   \n",
       "1    Linux kernel\\n============\\n\\nThere are severa...   \n",
       "..                                                 ...   \n",
       "16   \\n# [Flat UI Free](https://designmodo.github.i...   \n",
       "5    ![MuseScore](mscore/data/musescore_logo_full.p...   \n",
       "48   Hextris\\n==========\\n\\n<img src=\"images/twitte...   \n",
       "147  <iframe class=\"roundPhoto\" src=\"http://player....   \n",
       "36   <p align=\"center\">\\n  <img width=\"200\" src=\"sc...   \n",
       "\n",
       "                                               stemmed  \\\n",
       "26   about censu report the unit state censu bureau...   \n",
       "135  ngseed angularj requirej seed project document...   \n",
       "77   decis tree a rubi librari which implement id3 ...   \n",
       "84   div align center img src http www tensorflow o...   \n",
       "1    linux kernel there are sever guid for kernel d...   \n",
       "..                                                 ...   \n",
       "16   flat ui free http designmodo github io flat ui...   \n",
       "5    musescor mscore data musescor logo full png mu...   \n",
       "48   hextri img src imag twitter opengraph png widt...   \n",
       "147  ifram class roundphoto src http player vimeo c...   \n",
       "36   p align center img width 200 src screenshot tr...   \n",
       "\n",
       "                                            lemmatized  \\\n",
       "26   about census reporter the united state census ...   \n",
       "135  ngseed angularjs requirejs seed project docume...   \n",
       "77   decision tree a ruby library which implement i...   \n",
       "84   div align center img src http www tensorflow o...   \n",
       "1    linux kernel there are several guide for kerne...   \n",
       "..                                                 ...   \n",
       "16   flat ui free http designmodo github io flat ui...   \n",
       "5    musescore mscore data musescore logo full png ...   \n",
       "48   hextris img src image twitter opengraph png wi...   \n",
       "147  iframe class roundphoto src http player vimeo ...   \n",
       "36   p align center img width 200 src screenshots t...   \n",
       "\n",
       "                                                 clean  stopwords_removed  \\\n",
       "26   census reporter united state census bureau pro...               1573   \n",
       "135  ngseed angularjs requirejs seed documentation ...                115   \n",
       "77   decision tree ruby implement id3 information g...                101   \n",
       "84   div align center img src tensorflow image tf l...                513   \n",
       "1    linux kernel several guide kernel developer us...                 45   \n",
       "..                                                 ...                ...   \n",
       "16   flat ui free designmodo io flat ui flat ui bea...                219   \n",
       "5    musescore mscore data musescore logo full png ...                279   \n",
       "48   hextris img src image twitter opengraph png wi...                117   \n",
       "147  iframe class roundphoto src player vimeo video...                 57   \n",
       "36   p align center img width 200 src screenshots t...                188   \n",
       "\n",
       "     doc_length                                              words  \\\n",
       "26         2789  [census, reporter, united, state, census, bure...   \n",
       "135         265  [ngseed, angularjs, requirejs, seed, documenta...   \n",
       "77          264  [decision, tree, ruby, implement, id3, informa...   \n",
       "84         1586  [div, align, center, img, src, tensorflow, ima...   \n",
       "1            65  [linux, kernel, several, guide, kernel, develo...   \n",
       "..          ...                                                ...   \n",
       "16          392  [flat, ui, free, designmodo, io, flat, ui, fla...   \n",
       "5           583  [musescore, mscore, data, musescore, logo, ful...   \n",
       "48          148  [hextris, img, src, image, twitter, opengraph,...   \n",
       "147         127  [iframe, class, roundphoto, src, player, vimeo...   \n",
       "36          361  [p, align, center, img, width, 200, src, scree...   \n",
       "\n",
       "     unique_word_count  \n",
       "26                 861  \n",
       "135                143  \n",
       "77                 131  \n",
       "84                 324  \n",
       "1                   47  \n",
       "..                 ...  \n",
       "16                 179  \n",
       "5                  311  \n",
       "48                 103  \n",
       "147                 88  \n",
       "36                 214  \n",
       "\n",
       "[125 rows x 10 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can run a similar set of t-tests to compare the unique word count for each programming language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "language\n",
       "TypeScript          672.500000\n",
       "Python              480.368421\n",
       "Ruby                207.473684\n",
       "C#                  187.000000\n",
       "JavaScript          179.421053\n",
       "C++                 169.947368\n",
       "Scala               151.000000\n",
       "Java                129.562500\n",
       "C                   120.500000\n",
       "HTML                118.300000\n",
       "CSS                 102.500000\n",
       "Jupyter Notebook     71.500000\n",
       "Name: unique_word_count, dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('language').unique_word_count.mean().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------\n",
      "unique_word_count T-Test: TypeScript & Ruby\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of TypeScript and Ruby\n",
      "H_a: There is a difference in the mean unique_word_count of TypeScript and Ruby\n",
      "\n",
      "\n",
      "p-value: 0.006524370238190248\n",
      "t-statistic: 2.7387672495326547\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for TypeScript is larger than Ruby\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: TypeScript & JavaScript\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of TypeScript and JavaScript\n",
      "H_a: There is a difference in the mean unique_word_count of TypeScript and JavaScript\n",
      "\n",
      "\n",
      "p-value: 0.0026118583716749164\n",
      "t-statistic: 3.1541604406775816\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for TypeScript is larger than JavaScript\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: TypeScript & C++\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of TypeScript and C++\n",
      "H_a: There is a difference in the mean unique_word_count of TypeScript and C++\n",
      "\n",
      "\n",
      "p-value: 0.002243688809288501\n",
      "t-statistic: 3.2219830374586675\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for TypeScript is larger than C++\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: TypeScript & Java\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of TypeScript and Java\n",
      "H_a: There is a difference in the mean unique_word_count of TypeScript and Java\n",
      "\n",
      "\n",
      "p-value: 0.0018223481372979383\n",
      "t-statistic: 3.402007253476934\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for TypeScript is larger than Java\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: TypeScript & HTML\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of TypeScript and HTML\n",
      "H_a: There is a difference in the mean unique_word_count of TypeScript and HTML\n",
      "\n",
      "\n",
      "p-value: 0.003568499359874255\n",
      "t-statistic: 2.9961797832019097\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for TypeScript is larger than HTML\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Python & JavaScript\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Python and JavaScript\n",
      "H_a: There is a difference in the mean unique_word_count of Python and JavaScript\n",
      "\n",
      "\n",
      "p-value: 0.04780972198839981\n",
      "t-statistic: 1.7113584023481765\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Python is larger than JavaScript\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Python & C++\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Python and C++\n",
      "H_a: There is a difference in the mean unique_word_count of Python and C++\n",
      "\n",
      "\n",
      "p-value: 0.04297673614093736\n",
      "t-statistic: 1.7655451854834034\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Python is larger than C++\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Python & Java\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Python and Java\n",
      "H_a: There is a difference in the mean unique_word_count of Python and Java\n",
      "\n",
      "\n",
      "p-value: 0.03766515852093142\n",
      "t-statistic: 1.836339949265604\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Python is larger than Java\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Python & HTML\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Python and HTML\n",
      "H_a: There is a difference in the mean unique_word_count of Python and HTML\n",
      "\n",
      "\n",
      "p-value: 0.022513513053321135\n",
      "t-statistic: 2.074672099878915\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Python is larger than HTML\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Ruby & Java\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Ruby and Java\n",
      "H_a: There is a difference in the mean unique_word_count of Ruby and Java\n",
      "\n",
      "\n",
      "p-value: 0.03716912263055226\n",
      "t-statistic: 1.842926869750804\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Ruby is larger than Java\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: Ruby & Jupyter Notebook\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of Ruby and Jupyter Notebook\n",
      "H_a: There is a difference in the mean unique_word_count of Ruby and Jupyter Notebook\n",
      "\n",
      "\n",
      "p-value: 0.04462976975974244\n",
      "t-statistic: 1.7817420805404327\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for Ruby is larger than Jupyter Notebook\n",
      "\n",
      " \n",
      "\n",
      "----------------\n",
      "unique_word_count T-Test: JavaScript & Jupyter Notebook\n",
      "----------------\n",
      "Hypotheses:\n",
      "H_0: There is no difference in the mean unique_word_count of JavaScript and Jupyter Notebook\n",
      "H_a: There is a difference in the mean unique_word_count of JavaScript and Jupyter Notebook\n",
      "\n",
      "\n",
      "p-value: 0.04420442894931229\n",
      "t-statistic: 1.7868392716958894\n",
      "We reject the null hypothesis\n",
      "\n",
      "\n",
      "The mean readme unique_word_count for JavaScript is larger than Jupyter Notebook\n",
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "t_test_loop('unique_word_count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer()"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf = TfidfVectorizer()\n",
    "tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_validate, X_test are transformed to sparse array representing TF-IDF values for every word\n",
    "\n",
    "X_train = tfidf.fit_transform(train.clean)\n",
    "\n",
    "X_validate = tfidf.transform(validate.clean)\n",
    "\n",
    "X_test = tfidf.transform(test.clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_train, y_validate, y_test are simply the actual language labels\n",
    "\n",
    "y_train = train.language\n",
    "y_validate = validate.language\n",
    "y_test = test.language"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline\n",
    "\n",
    "We will create two baseline models. \n",
    "\n",
    "The first will predict a language at random using weighted probabilities based on the distribution of the languages in the train dataset. \n",
    "\n",
    "The second will be based on the most common language in the train dataset.\n",
    "\n",
    "### Random Baseline Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def baseline_language_model(train, num_observations):\n",
    "    '''\n",
    "    This function takes in a train dataset and identifies the proportions for the languages represented in that set.\n",
    "    It then creates an array num_observations long where each element is one of the languages randomly selected (weighted by the languages proportion)\n",
    "    The array is returned to be used as a series of predictions to compare other models against.\n",
    "    '''\n",
    "    possible_languages = list(train.language.value_counts().index)\n",
    "    probabilities = []\n",
    "    for language in possible_languages:\n",
    "        probabilities.append(train.language.value_counts()[language] / train.language.shape[0])\n",
    "    baseline_predictions = choice(possible_languages, num_observations, p=probabilities)\n",
    "    return baseline_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Jupyter Notebook', 'Ruby', 'Python', 'Java', 'Java', 'Python',\n",
       "       'HTML', 'C++', 'Python', 'HTML'], dtype='<U16')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_train_pred_random = baseline_language_model(train, num_observations = train.language.shape[0])\n",
    "baseline_train_pred_random[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C++</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>CSS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C++</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C++</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random\n",
       "26         HTML  Jupyter Notebook\n",
       "135        HTML              Ruby\n",
       "77         Ruby            Python\n",
       "84          C++              Java\n",
       "1             C              Java\n",
       "..          ...               ...\n",
       "16   JavaScript               CSS\n",
       "5           C++            Python\n",
       "48   JavaScript            Python\n",
       "147         C++              Ruby\n",
       "36   JavaScript            Python\n",
       "\n",
       "[125 rows x 2 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results = pd.DataFrame(dict(actual=y_train))\n",
    "train_results['base_random'] = baseline_train_pred_random\n",
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Weighted Random Selection\n",
      "Accuracy: 15.20%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual            C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  \\\n",
      "base_random                                                                   \n",
      "C                 0   0    1    0     0     0           1                 0   \n",
      "C#                0   0    1    0     0     0           0                 0   \n",
      "C++               0   0    0    1     3     5           3                 1   \n",
      "CSS               0   0    0    0     2     0           1                 0   \n",
      "HTML              0   1    3    0     4     2           2                 0   \n",
      "Java              1   0    4    1     0     3           2                 0   \n",
      "JavaScript        0   0    3    0     3     3           3                 0   \n",
      "Jupyter Notebook  0   0    1    0     1     0           1                 0   \n",
      "Python            0   1    5    0     2     1           4                 1   \n",
      "Ruby              1   0    1    0     5     2           2                 2   \n",
      "TypeScript        0   0    0    0     0     0           0                 0   \n",
      "\n",
      "actual            Python  Ruby  Scala  TypeScript  \n",
      "base_random                                        \n",
      "C                      0     0      0           0  \n",
      "C#                     0     0      0           0  \n",
      "C++                    3     2      0           0  \n",
      "CSS                    0     1      0           0  \n",
      "HTML                   0     3      0           0  \n",
      "Java                   4     0      0           0  \n",
      "JavaScript             0     2      0           0  \n",
      "Jupyter Notebook       2     0      0           0  \n",
      "Python                 7     8      1           2  \n",
      "Ruby                   3     2      0           0  \n",
      "TypeScript             0     1      0           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         2\n",
      "              C#       0.00      0.00      0.00         2\n",
      "             C++       0.00      0.00      0.00        19\n",
      "             CSS       0.00      0.00      0.00         2\n",
      "            HTML       0.27      0.20      0.23        20\n",
      "            Java       0.20      0.19      0.19        16\n",
      "      JavaScript       0.21      0.16      0.18        19\n",
      "Jupyter Notebook       0.00      0.00      0.00         4\n",
      "          Python       0.22      0.37      0.27        19\n",
      "            Ruby       0.11      0.11      0.11        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         2\n",
      "\n",
      "        accuracy                           0.15       125\n",
      "       macro avg       0.08      0.08      0.08       125\n",
      "    weighted avg       0.15      0.15      0.15       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Weighted Random Selection')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.base_random)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.base_random, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.base_random))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Model: Mode Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C++</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>CSS</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C++</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C++</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode\n",
       "26         HTML  Jupyter Notebook      HTML\n",
       "135        HTML              Ruby      HTML\n",
       "77         Ruby            Python      HTML\n",
       "84          C++              Java      HTML\n",
       "1             C              Java      HTML\n",
       "..          ...               ...       ...\n",
       "16   JavaScript               CSS      HTML\n",
       "5           C++            Python      HTML\n",
       "48   JavaScript            Python      HTML\n",
       "147         C++              Ruby      HTML\n",
       "36   JavaScript            Python      HTML\n",
       "\n",
       "[125 rows x 3 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results['base_mode'] = train.language.value_counts().idxmax()\n",
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Mode Selection\n",
      "Accuracy: 16.00%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual     C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "base_mode                                                                      \n",
      "HTML       2   2   19    2    20    16          19                 4      19   \n",
      "\n",
      "actual     Ruby  Scala  TypeScript  \n",
      "base_mode                           \n",
      "HTML         19      1           2  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         2\n",
      "              C#       0.00      0.00      0.00         2\n",
      "             C++       0.00      0.00      0.00        19\n",
      "             CSS       0.00      0.00      0.00         2\n",
      "            HTML       0.16      1.00      0.28        20\n",
      "            Java       0.00      0.00      0.00        16\n",
      "      JavaScript       0.00      0.00      0.00        19\n",
      "Jupyter Notebook       0.00      0.00      0.00         4\n",
      "          Python       0.00      0.00      0.00        19\n",
      "            Ruby       0.00      0.00      0.00        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         2\n",
      "\n",
      "        accuracy                           0.16       125\n",
      "       macro avg       0.01      0.08      0.02       125\n",
      "    weighted avg       0.03      0.16      0.04       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Mode Selection')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.base_mode)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.base_mode, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.base_mode))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = LogisticRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C++</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>CSS</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C++</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C++</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode lm_predicted\n",
       "26         HTML  Jupyter Notebook      HTML         HTML\n",
       "135        HTML              Ruby      HTML         HTML\n",
       "77         Ruby            Python      HTML         Ruby\n",
       "84          C++              Java      HTML          C++\n",
       "1             C              Java      HTML       Python\n",
       "..          ...               ...       ...          ...\n",
       "16   JavaScript               CSS      HTML   JavaScript\n",
       "5           C++            Python      HTML          C++\n",
       "48   JavaScript            Python      HTML   JavaScript\n",
       "147         C++              Ruby      HTML          C++\n",
       "36   JavaScript            Python      HTML   JavaScript\n",
       "\n",
       "[125 rows x 4 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results['lm_predicted'] = lm.predict(X_train)\n",
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 89.60%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual        C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  \\\n",
      "lm_predicted                                                              \n",
      "C++           1   0   19    0     0     0           0                 1   \n",
      "HTML          0   2    0    1    20     0           0                 0   \n",
      "Java          0   0    0    0     0    16           0                 0   \n",
      "JavaScript    0   0    0    1     0     0          19                 0   \n",
      "Python        1   0    0    0     0     0           0                 2   \n",
      "Ruby          0   0    0    0     0     0           0                 1   \n",
      "\n",
      "actual        Python  Ruby  Scala  TypeScript  \n",
      "lm_predicted                                   \n",
      "C++                0     0      0           0  \n",
      "HTML               0     0      1           0  \n",
      "Java               0     0      0           0  \n",
      "JavaScript         0     0      0           2  \n",
      "Python            19     0      0           0  \n",
      "Ruby               0    19      0           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         2\n",
      "              C#       0.00      0.00      0.00         2\n",
      "             C++       0.90      1.00      0.95        19\n",
      "             CSS       0.00      0.00      0.00         2\n",
      "            HTML       0.83      1.00      0.91        20\n",
      "            Java       1.00      1.00      1.00        16\n",
      "      JavaScript       0.86      1.00      0.93        19\n",
      "Jupyter Notebook       0.00      0.00      0.00         4\n",
      "          Python       0.86      1.00      0.93        19\n",
      "            Ruby       0.95      1.00      0.97        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         2\n",
      "\n",
      "        accuracy                           0.90       125\n",
      "       macro avg       0.45      0.50      0.47       125\n",
      "    weighted avg       0.81      0.90      0.85       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.lm_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.lm_predicted, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.lm_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = DecisionTreeClassifier(max_depth = 15, random_state = 123).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['HTML', 'HTML', 'Ruby', 'C++', 'C', 'HTML', 'C++', 'Java',\n",
       "       'TypeScript', 'Ruby'], dtype=object)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_pred = clf.predict(X_train)\n",
    "clf_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "      <th>clf_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C++</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>CSS</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C++</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C++</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode lm_predicted clf_predicted\n",
       "26         HTML  Jupyter Notebook      HTML         HTML          HTML\n",
       "135        HTML              Ruby      HTML         HTML          HTML\n",
       "77         Ruby            Python      HTML         Ruby          Ruby\n",
       "84          C++              Java      HTML          C++           C++\n",
       "1             C              Java      HTML       Python             C\n",
       "..          ...               ...       ...          ...           ...\n",
       "16   JavaScript               CSS      HTML   JavaScript    JavaScript\n",
       "5           C++            Python      HTML          C++           C++\n",
       "48   JavaScript            Python      HTML   JavaScript    JavaScript\n",
       "147         C++              Ruby      HTML          C++           C++\n",
       "36   JavaScript            Python      HTML   JavaScript    JavaScript\n",
       "\n",
       "[125 rows x 5 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results['clf_predicted'] = clf_pred\n",
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Model\n",
      "Accuracy: 94.40%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual            C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  \\\n",
      "clf_predicted                                                                 \n",
      "C                 2   0    0    0     0     0           0                 0   \n",
      "C#                0   2    0    0     0     0           0                 0   \n",
      "C++               0   0   17    0     0     0           0                 0   \n",
      "CSS               0   0    0    2     0     0           0                 1   \n",
      "HTML              0   0    2    0    20     0           0                 2   \n",
      "Java              0   0    0    0     0    16           0                 0   \n",
      "JavaScript        0   0    0    0     0     0          19                 0   \n",
      "Jupyter Notebook  0   0    0    0     0     0           0                 1   \n",
      "Python            0   0    0    0     0     0           0                 0   \n",
      "Ruby              0   0    0    0     0     0           0                 0   \n",
      "TypeScript        0   0    0    0     0     0           0                 0   \n",
      "\n",
      "actual            Python  Ruby  Scala  TypeScript  \n",
      "clf_predicted                                      \n",
      "C                      0     0      0           0  \n",
      "C#                     0     0      0           0  \n",
      "C++                    0     0      0           0  \n",
      "CSS                    0     0      0           0  \n",
      "HTML                   0     0      0           0  \n",
      "Java                   0     0      0           0  \n",
      "JavaScript             0     0      0           0  \n",
      "Jupyter Notebook       0     0      0           0  \n",
      "Python                19     1      1           0  \n",
      "Ruby                   0    18      0           0  \n",
      "TypeScript             0     0      0           2  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       1.00      1.00      1.00         2\n",
      "              C#       1.00      1.00      1.00         2\n",
      "             C++       1.00      0.89      0.94        19\n",
      "             CSS       0.67      1.00      0.80         2\n",
      "            HTML       0.83      1.00      0.91        20\n",
      "            Java       1.00      1.00      1.00        16\n",
      "      JavaScript       1.00      1.00      1.00        19\n",
      "Jupyter Notebook       1.00      0.25      0.40         4\n",
      "          Python       0.90      1.00      0.95        19\n",
      "            Ruby       1.00      0.95      0.97        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       1.00      1.00      1.00         2\n",
      "\n",
      "        accuracy                           0.94       125\n",
      "       macro avg       0.87      0.84      0.83       125\n",
      "    weighted avg       0.95      0.94      0.93       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Decision Tree Model')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.clf_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.clf_predicted, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.clf_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(bootstrap=True, \n",
    "                            class_weight=None, \n",
    "                            criterion='gini',\n",
    "                            min_samples_leaf=3,\n",
    "                            n_estimators=100,\n",
    "                            max_depth=3, \n",
    "                            random_state=123).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['HTML', 'HTML', 'Ruby', 'C++', 'HTML', 'HTML', 'Python', 'Java',\n",
       "       'Python', 'Ruby'], dtype=object)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_pred = rf.predict(X_train)\n",
    "rf_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_results['rf_predicted'] = rf_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Model\n",
      "Accuracy: 80.00%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual        C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  \\\n",
      "rf_predicted                                                              \n",
      "C++           0   0   13    0     0     0           0                 0   \n",
      "HTML          1   2    5    2    19     1           4                 3   \n",
      "Java          0   0    0    0     1    15           0                 0   \n",
      "JavaScript    1   0    0    0     0     0          15                 0   \n",
      "Python        0   0    1    0     0     0           0                 1   \n",
      "Ruby          0   0    0    0     0     0           0                 0   \n",
      "\n",
      "actual        Python  Ruby  Scala  TypeScript  \n",
      "rf_predicted                                   \n",
      "C++                0     0      0           0  \n",
      "HTML               0     0      0           0  \n",
      "Java               0     0      0           0  \n",
      "JavaScript         0     0      1           1  \n",
      "Python            19     0      0           1  \n",
      "Ruby               0    19      0           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         2\n",
      "              C#       0.00      0.00      0.00         2\n",
      "             C++       1.00      0.68      0.81        19\n",
      "             CSS       0.00      0.00      0.00         2\n",
      "            HTML       0.51      0.95      0.67        20\n",
      "            Java       0.94      0.94      0.94        16\n",
      "      JavaScript       0.83      0.79      0.81        19\n",
      "Jupyter Notebook       0.00      0.00      0.00         4\n",
      "          Python       0.86      1.00      0.93        19\n",
      "            Ruby       1.00      1.00      1.00        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         2\n",
      "\n",
      "        accuracy                           0.80       125\n",
      "       macro avg       0.43      0.45      0.43       125\n",
      "    weighted avg       0.76      0.80      0.77       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Random Forest Model')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.rf_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.rf_predicted, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.rf_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_rate = []\n",
    "for i in range(1,40):\n",
    "    \n",
    "    knn = KNeighborsClassifier(n_neighbors=i)\n",
    "    knn.fit(X_train,y_train)\n",
    "    pred_i = knn.predict(X_test)\n",
    "    error_rate.append(np.mean(pred_i != y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Error Rate')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAGDCAYAAABuj7cYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABEFklEQVR4nO3deXxU9b3/8dcnC0uAgJqIIoIk1bYW0cqooF201lase1uvctXWqoBIr1Ktld6u99d6bal6S11AccNe67WuaEEtbdW2iDW44UI1oYCIylaRhBBD8vn9cSYlhMlkZjIzZ5b38/GYB5kz5zvnnTOTmQ/nnO/3a+6OiIiIiOSGkrADiIiIiMgOKs5EREREcoiKMxEREZEcouJMREREJIeoOBMRERHJISrORERERHKIijMRkQJmZneY2U/CziEiiVNxJiJxmdlKM2s2s8ZOt+uznOFJM9sW3fYGM3vAzPZOsO3RZrYm0xmTYWb7mZmbWVn0vpnZr8xsuZnt02Xds6KvgXVZXmZm68zsxGxmF5HMU3EmIok4yd0HdrpNi7VSR7HRZVlpMhuKs/40dx8IfAQYCPwimefNVdGiaw5wNPBZd3+7yyoPAkOAz3ZZfjzgwGMZjigiWabiTERSZmZfN7O/mtl1ZrYJ+FH0NNpNZrbAzJqAY8zs49GjX++b2atmdnKn59hl/XjbdPf3gYeAQzo9x3lm9rqZbTGzFWY2Obp8ALAQGNbpqN8wMysxsyvNrMHMNprZvWa2eze/4+udj05Fj1htMLNDzayfmf06+hzvm9lzZjY0iV1YCtwBRICj3f29GL/vNuBe4NwuD50L/K+7bzez35rZu2a22cyeNrNPdPO7fN3M/tJlmZvZR6I/9zWzX5jZajN7z8xmm1n/JH4fEUkDFWci0ltHACuAPYGfRpdNjP48CHgWeAR4IrrON4H/NbOPdnqOzuvvVDx0ZWZ7AKcD9Z0WrwNOBCqB84DrzOxQd28CJgBrOx31Wwv8B3AqwdGoYcA/gRu62eRvgLM63f8isMHdnwe+BgwG9gX2AKYAzfHyd/G/wMeAz7n7xjjr3Ql8paNQMrPBwEnAvOjjC4H9Cfbv89HnTcXPgAMICt+PAPsAP0jxuUQkRSrORCQRD0WPDHXcLuz02Fp3/5W7b3f3jsLkYXf/q7u3E3zRDwSudvcP3f2PwKPsXPD8a/3okaJYZpnZZmADUEVQ5AHg7r9z9wYPPEVQCH46zu8zGfhPd1/j7i3AjwiKn11OywJ3AyebWUX0/sToMoBWgqLsI+7e5u5L3f2DONvt6gvAvdGjgd1y978C7wGnRRedAbzh7i9GH7/N3bd0+l0OjhZwCYueXr0QmO7um9x9C3AVcGYyzyMivafiTEQScaq7D+l0u6XTY2/FWL/zsmHAW9FCrcMqgqMy8Z6jq/9w98HAGGA3YHjHA2Y2wcyWmNkmM3sfOIGggOvOSODBjmITeB1oA3Y5Jenu9dHHT4oWaCezozi7C3gcuMfM1prZz82sPIHfpcOJwA/N7BsJrDuPHac2zyE4moaZlZrZ1dFTtB8AK6PrxPv9Y6kGKoClnfbLY9HlIpJFKs5EpLe8h2VrgX3NrPPnzQjg7W7Wj78x92XAT4Abor0c+wL3E3QQGOruQ4AFQEfvxljP/RYwoUvB2S/GxfgdOk5tngK8Fi3YcPdWd/+xux8IHElQbHW9NiyexQSnJ39pZhN7WHcecKyZjQfGsaNAnBjN9XmCU6z7RZdb1ycAmggKsGAFs706PbaB4JTsJzrtk8HRThgikkUqzkQk054lKAquMLNyMzuaoCC5pxfPeSfB9VUnA32AvsB6YLuZTSA4XdjhPWCPLqf5ZgM/NbORAGZWbWanxNnePdHnvIgdRRFmdoyZHRTtYfoBwWnOtmR+kehp2NOBm83sK3HWW0VwPd5vgN+7+7vRhwYBLcBGgsLrqjibewn4hJkdYmb9CE6Bdjx/O3ALwfV6e0Z/v33M7IvJ/D4i0nsqzkQkEY/YzuOcPZhoQ3f/kKCImkBwdOZG4Fx3X55qmOhzzgK+H7026j8IejT+k+BI0vxO6y4nKGhWRE/XDQN+GV3nCTPbAiwh6NjQ3fbeAZ4hODr2f50e2gu4j6Awex14Cvg1QLSn4+wEf5/fA/8G3GFmJ8VZ9U6CU7LzOi2bR3Ca+G3gtejv0t123gD+C1gEvMmunS++Q9DRYkn0FOki4KOISFaZe8JnE0REREQkw3TkTERERCSHqDgTERERySEqzkRERERySEaLMzM73sz+bmb1ZnZljMcHm9kjZvZSdEqX8xJtKyIiIlKIMtYhINq1/A3gOGAN8Bxwlru/1mmd7wKD3f07ZlYN/J2g91NbT21FREREClGsqUrS5XCg3t1XAJjZPUQHcOy0jgODotOGDAQ2AdsJurT31HYXVVVVvt9++6X51xARERFJv6VLl25w911m4chkcbYPO0/JsoZdxxG6nmCsobUEAyn+m7u3m1kibQEws0nAJIARI0ZQV1eXnvQiIiIiGWRmq2Itz+Q1Z7GmDul6DvWLwIsEc+8dAlxvZpUJtg0Wut/s7hF3j1RXawo4ERERyW+ZLM7WAPt2uj+c4AhZZ+cBD3igHvgH8LEE24qIiIgUnEwWZ88B+5vZKDPrA5xJpylVolYDxwKY2VCCaUJWJNhWREREpOBk7Jozd99uZtOAx4FS4DZ3f9XMpkQfnw38P4K55JYRnMr8jrtvAIjVNlNZRURERHJFQc2tGYlEXB0CREREJB+Y2VJ3j3RdrhkCRERERHKIijMRERGRHKLiLAENDTB9agtDK5spLWlnaGUz06e20NAQdjIREREpNCrOerBwIYwb00T/ubNYvGU0Ld6HxVtG03/uLMaNaWLhwrATioiISCFRh4A4GhqCwmz+1s8zniW7PP4M4zi5YhFLXh5AbW3aNisiIiJFQB0CUnD9NS1c2HpjzMIMYDxLuKD1Jm64riXLyURERKRQqTiL4+5ft3N+6+y461zQehN339WWpUQiIiJS6FScxbGhsS8jiTkn6b+MYDUbGvtlKZGIiIgUOhVncVQNbGEVI+Ous5oRVA3clqVEIiIiUuhUnMUx8ewSbi2fEnedueUXMfGc0iwlEhERkUKn4iyOaZf15ZbyqTzDuJiPP8M45pZfxMXT+2Y5mYiIiBQqFWdx1NbCvPsGcHLFImaUz6SBGlopo4EaZpTP5OSKRcy7T8NoiIiISPqoOOvBhAmw5OUBtEz6JuMHLqMvLRzefxktk77JkpcHMGFC2AlFRESkkGgQ2iS0t4M7lOoSMxEREeml7gahLQsjTL4q0XFGERERyTCVG0maNQsuuSTsFCIiIlKoVJwl6ZVX4De/CTuFiIiIFCoVZ0mqqYH162HLlrCTiIiISCFScZakjmEzVqwIN4eIiIgUJhVnSeoozhoaws0hIiIihUnFWZJqa2HECPjww7CTiIiISCHSUBpJGjwYVq0KO4WIiIgUKh05ExEREckhKs5S8POfw/HHh51CRERECpGKsxRs3gx/+ANs3x52EhERESk0Ks5SUFMTFGZvvRV2EhERESk0Ks5SoOE0REREJFNUnKWgpib4V8WZiIiIpJuKsxTssw989rMwZEjYSURERKTQaJyzFJSWwpNPhp1CRERECpGOnImIiIjkEBVnKbrqKhg+HNzDTiIiIiKFRMVZivr3h7ffho0bw04iIiIihUTFWYo6htNYsSLcHCIiIlJYVJylSMNpiIiISCaoOEtRR3GmI2ciIiKSTirOUlRRAZMmwYEHhp1ERERECklGxzkzs+OBXwKlwFx3v7rL498G/r1Tlo8D1e6+ycxWAluANmC7u0cymTUVc+aEnUBEREQKTcaKMzMrBW4AjgPWAM+Z2Xx3f61jHXefCcyMrn8SMN3dN3V6mmPcfUOmMqbD5s0weHDYKURERKRQZPK05uFAvbuvcPcPgXuAU+KsfxbwmwzmSburr4bddoNt28JOIiIiIoUik8XZPsBbne6viS7bhZlVAMcD93da7MATZrbUzCZ1txEzm2RmdWZWt379+jTETlzHILQrV2Z1syIiIlLAMlmcWYxl3Y2nfxLw1y6nNI9y90OBCcDFZvaZWA3d/WZ3j7h7pLq6uneJk6SxzkRERCTdMlmcrQH27XR/OLC2m3XPpMspTXdfG/13HfAgwWnSnKKxzkRERCTdMlmcPQfsb2ajzKwPQQE2v+tKZjYY+CzwcKdlA8xsUMfPwBeAVzKYNSV77gkDBujImYiIiKRPxnpruvt2M5sGPE4wlMZt7v6qmU2JPj47uuppwBPu3tSp+VDgQTPryHi3uz+WqaypMoP/+i+NdSYiIiLpY+7dXQaWfyKRiNfV1YUdQ0RERKRHZrY01jiumiGgl7ZuhWXLoL097CQiIiJSCFSc9dIdd8CYMfDuu2EnERERkUKg4qyXOobTUI9NERERSQcVZ72k4TREREQknVSc9dLIkVBSouE0REREJD1UnPVSnz6w7746ciYiIiLpkbFxzorJddfB0KFhpxAREZFCoOIsDU47LewEIiIiUih0WjMN1q+HBQuguTnsJCIiIpLvVJylwZ/+BF/6ErzxRthJREREJN+pOEuDjrHO1GNTREREekvFWRpoIFoRERFJFxVnaTBkCOy2m46ciYiISO+pOEuT2lodORMREZHe01AaaXLjjVBZGXYKERERyXcqztLksMPCTiAiIiKFQKc10+Ttt2HOHNiwIewkIiIiks9UnKXJG2/AlCnw0kthJxEREZF8puIsTTSchoiIiKSDirM02WcfKC9XcSYiIiK9o+IsTUpLYdQojXUmIiIivaPiLI001pmIiIj0lobSSKObb4YBA8JOISIiIvlMxVkaDR8edgIRERHJdzqtmUarV8P3vgf19WEnERERkXyl4iyNPvgAfvpTqKsLO4mIiIjkKxVnaTRqVPCvOgWIiIhIqlScpdGAAbDXXhpOQ0RERFKn4izNamp05ExERERSp+IszWpr4d13w04hIiIi+UpDaaTZnDnQr1/YKURERCRfqThLs/79w04gIiIi+UynNdPs7bfha1+DZ54JO4mIiIjkIxVnaVZWBvPmwdKlYScRERGRfKTiLM323DMYUkM9NkVERCQVKs7SzCwYTkNjnYmIiEgqVJxlgMY6ExERkVRltDgzs+PN7O9mVm9mV8Z4/Ntm9mL09oqZtZnZ7om0zWUHHgh9+4adQkRERPKRuXtmntisFHgDOA5YAzwHnOXur3Wz/knAdHf/XLJtO0QiEa/TrOMiIiKSB8xsqbtHui7P5JGzw4F6d1/h7h8C9wCnxFn/LOA3KbYVERERKQiZLM72Ad7qdH9NdNkuzKwCOB64P9m2uWj9evjc5+CBB8JOIiIiIvkmk8WZxVjW3TnUk4C/uvumZNua2SQzqzOzuvXr16cQM/0GD4Ynn4SXXgo7iYiIiOSbTBZna4B9O90fDqztZt0z2XFKM6m27n6zu0fcPVJdXd2LuOnTpw/su696bIqIiEjyMlmcPQfsb2ajzKwPQQE2v+tKZjYY+CzwcLJtc1ltrYozERERSV7GijN33w5MAx4HXgfudfdXzWyKmU3ptOppwBPu3tRT20xlzYTaWg1EKyIiIskry+STu/sCYEGXZbO73L8DuCORtvnksMNg9WpobYXy8rDTiIiISL7QDAEZMmkSPP64CjMRERFJjoozERERkRyi4ixDtm6F/feHWbPCTiIiIiL5RMVZhlRUwMaNsHx52ElEREQkn6g4yyD12BQREZFkqTjLoJoajXUmIiIiyVFxlkG1tbByJbS1hZ1ERERE8kVGxzkrdp/6FLz3XtA5YNCgsNOIiIhIPlBxlkEnnBDcRERERBKl05oZ0tAA06e2MLSymdKSdoZWNjN9aouuQRMREZG4VJxlwMKFMG5ME/3mzmLxltG0eB8WbxlN/7mzGDemiYULw04oIiIiucrcPewMaROJRLyuri7UDA0NQWE2f+vnGc+SXR5/hnGcXLGIJS8PoLY2hIAiIiKSE8xsqbtHui7XkbM0u/6aFi5svTFmYQYwniVc0HoTN1zXkuVkIiIikg9UnKXZ3b9u5/zW2XHXuaD1Ju6+S+NriIiIyK5UnKXZhsa+jGRV3HVGsJoNjf2ylEhERETyiYqzNKsa2MIqRsZdZzUjqBq4LUuJREREJJ+oOEuziWeXcGv5lLjrzC2/iInnlGYpkYiIiOQTFWdpNu2yvtxSPpVnGBfz8WcYx9zyi7h4et8sJxMREZF8oOIszWprYd59Azi5YhEzymfSQA2tlNFADZczk5MrFjHvPg2jISIiIrFp+qYMmDABlrw8gBuu+yZH3TWVDY39qOyzjcZtpdz7v32ZMCHshCIiIpKrdOQsQ2pr4drr+/Lu5gq2t5Xwj3crKK/oy+9+F3YyERERyWUqzrJk8GCYOBHuvhs2bw47jYiIiOQqFWdZNHky7LEHvPlm2ElEREQkV+masyyKROAf/4BSjaIhIiIi3dCRsywrLYUPP4QNG8JOIiIiIrlIxVmWtbfDQQfB9OlhJxEREZFcpOIsy0pK4POfh9/+FjZuDDuNiIiI5BoVZyGYPBlaWuDOO8NOIiIiIrlGxVkIxoyBI4+EOXPAPew0IiIikktUnIVk8mR44w3461/DTiIiIiK5RENphOSrX4URI+Coo8JOIiIiIrlExVlI+veHo48OO4WIiIjkGp3WDFFbG1x6KVx/fdhJREREJFeoOAtRaSm89BJcc00w/pmIiIiIirOQTZkCK1fCE0+EnURERERygYqzkJ12GlRXB8NqiIiIiKg4C1mfPnD++fDII7BmTdhpREREJGwZ7a1pZscDvwRKgbnufnWMdY4G/gcoBza4+2ejy1cCW4A2YLu7RzKZNUwXXgjvvRd0EBAREZHilrHizMxKgRuA44A1wHNmNt/dX+u0zhDgRuB4d19tZnt2eZpj3H1DpjLmipoauO22sFOIiIhILsjkac3DgXp3X+HuHwL3AKd0WWci8IC7rwZw93UZzJPz6urg+efDTiEiIiJh6rE4s8DZZvaD6P0RZnZ4As+9D/BWp/tross6OwDYzcyeNLOlZnZup8cceCK6fFIC28tr27fDKafA974XdhIREREJUyJHzm4ExgNnRe9vIThd2ROLsazrNN9lwFjgS8AXge+b2QHRx45y90OBCcDFZvaZmBsxm2RmdWZWt379+gRi5aayMrjgAnjssWBoDRERESlOiRRnR7j7xcA2AHf/J9AngXZrgH073R8OrI2xzmPu3hS9tuxp4ODodtZG/10HPEhwmnQX7n6zu0fcPVJdXZ1ArNx1wQXBv2d9uYWhlc2UlrQztLKZ6VNbaGgIN5uIiIhkRyLFWWv04n4HMLNqIJHx7J8D9jezUWbWBzgTmN9lnYeBT5tZmZlVAEcAr5vZADMbFN3eAOALwCsJ/UZ57JVXYGBJE596fhaLt4ymxfuweMto+s+dxbgxTSxcGHZCERERybREemvOIjhytaeZ/RT4CvD9nhq5+3YzmwY8TjCUxm3u/qqZTYk+PtvdXzezx4CXCQq+ue7+ipnVAA+aWUfGu939sRR+v7zR0ADnfqWJx9s+z3iW/Gt5LSu4qvUKTmp9gJO/soglLw+gtjbEoCIiIpJR5t71MrAYK5l9DDiW4DqyP7j765kOlopIJOJ1dXVhx0jJ9Kkt9J87i6tar+h2nRnlM2mZ9E2uvb5vFpOJiIhIJpjZ0ljjuPZYnJnZXe5+Tk/LckE+F2dDK5tZvGU0tazodp0Gajiqchnvbq7IYjIRERHJhO6Ks0SuOftElycqJehhKWm0obEvI1kVd50RrGZDY78sJRIREZEwdFucmdkMM9sCjDGzD8xsS/T+OoIL+SWNqga2sIqRcddZzQiqBm7LUiIREREJQ7fFmbv/t7sPAma6e6W7D4re9nD3GVnMWBQmnl3CreVT4q4zt/wiJp5TmqVEIiIiEoZEOwTsBuwP/Oucmrs/ncFcKcnna84aGmDcmCbmb925t2aHZxjHyRXqrSkiIlIoUr7mzMwuIBgc9nHgx9F/f5TugMWuthbm3TeAkysWMaN8Jg3U0EoZDdQwo3wmJ1csYt59KsxEREQKXSIdAi4BDgNWufsxwCeB/J0nKYdNmABLXh5Ay6RvcuSgZfSjhfEDltEy6ZsseXkAEyaEnVBEREQyLZFBaLe5+zYzw8z6uvtyM/toxpMVqdpauPb6vlx7fccSDZshIiJSTBI5crbGzIYADwG/N7OH2XWOTMmA9nZ4882wU4iIiEg29Vicuftp7v6+u/+IYNqmW4FTMh1M4Npr4YADYMOGsJOIiIhItiRy5Oxf3P0pYBuwIDNxpLNDDw3+Xbo03BwiIiKSPfEGof2cmb1hZo1m9mszO9DM6oD/Bm7KXsTi1VGc5enoICIiIpKCeEfOrgEmAXsA9wFLgLvcfay7P5CNcMVuyBDYf38dORMRESkm8Yozd/cn3b3F3R8C1rv7L7OUS6IiER05ExERKSbxhtIYYmand7pvne/r6Fl2TJsGZ54J7mAWdhoRERHJtHjF2VPASd3cd0DFWRYceWTYCURERCSbui3O3P28bAaR7v35z1BWBuPHh51EREREMi2RGQIkZJMnw0c+AvPnh51EREREMi2pcc4kHJGIemyKiIgUi7jFmZmVmJmuegpZJAJr1wY3ERERKWxxizN3bycY70xCFIkE/+romYiISOFL5LTmE2b2ZTMN5BCWQw6BkhJ47rmwk4iIiEimJdIh4FvAAKDNzJoBIxigtjKjyeRfKirg+eeDSdBFRESksPVYnLn7oGwEkfgOPjjsBCIiIpINCfXWNLOTzewX0duJmQ4lu3rzTbjiCnjnnbCTiIiISCb1WJyZ2dXAJcBr0dsl0WWSRZs2wcyZsGRJ2ElEREQkkxI5cnYCcJy73+butwHHR5dJFo0ZE8wSoB6bIiIihS3RQWiHdPp5cAZySA/694fRo6GuLuwkIiIikkmJ9Na8CnjBzP5E0FPzM8CMjKaSmMaOhYceAnfQwCYiIiKFqccZAoB2YBzwQPQ23t3vyUI26aJjMNr163v/XA0NMH1qC0MrmyktaWdoZTPTp7bQ0ND75xYRKUSpfG6m+lmbrW1lM1825Xq+niQyQ8A0d3/H3ee7+8Pu/m6WskkX558fFGZ77tm751m4EMaNaaL/3Fks3jKaFu/D4i2j6T93FuPGNLFwYXryiogUilQ+N1P9rM3WtrKZL5tyPV9C3D3uDfg+cDmwL7B7x62ndmHcxo4d6xJffb17VUWjL2ace3CGdKfbYsZ5VUWj19eHnVREJDek8rmZ6mdttraVzXzZlOv5ugLqPEY9k0iHgG8AFwNPA0ujN12WHpIf/xgmTUq9/fXXtHBh642MJ/aYHONZwgWtN3HDdS2pb0REpICk8rmZ6mdttraVzXzZlOv5EharYuu4EZz2/Ld46+TSrRiOnE2e7D5kiHt7e2rt9xy01eupifk/io5bPTU+tLIpvcFFRPJUop+bg/s0+aWXul96qXtln+Q+ax96KLl2g/s0+RVXZC/fNdek1i5XX6tc+Y6jmyNnFjzWPTN72t0/k4U6sdcikYjXFfhYE3PnwoUXQn091NYm3760pJ0W70MZbd2u00oZ/Uta2N6W6EgrIiKFK9HPzX60MLAy+Nzc8kE7H5LAZ621sL29hO99D371q8Tb9aOFoXuXsHZthvNFvws++1l48cXk22Vbvn3HmdlSd490XZ5Ist+b2eVmtq+Z7d5xy0BGScDYscG/qdagVQNbWMXIuOusZgRVA7eltgERkQKT6OdmdeU2Nm+GzZuhelCCn7WDgs/an/wkuXbVldtYuzYL+aLfBU89lVq7bCuU7zhdc5ZnPvEJ6Ns39eJs4tkl3Fo+Je46c8svYuI5paltQESkwKTyuZnqZ222tpXNfNmU6/kSFutcZ7puBFM9/R2oB67sZp2jgReBV4Gnkmnb9VYM15y5u591lvvVV6fWNt96soiIhC3Xe0Pmer5syvV8XdHNNWfxCqsrOv381S6PXdVdu07rlAINQA3QB3gJOLDLOkMIJlMfEb2/Z6JtY92KpTjrrQUL3Hfv1+jfYqbXU+MfUub11PiV5TO9qqLRFywIO6GISG5ZsCD40r+yPPHPzVTaZHNb6c737dLc+A7pNl9JbuTrLJXi7PlYP8e630378cDjne7PAGZ0WWcq8JNU2sa6FVNx1t7uvn176u1PO819QPk233NQk5fQ5pXlTT794m05878JEZFcsn27+/Tp7hecu82HVjZ5aUmbD63s+XOzvt59+sXJtUm1XbbaxGo3oKTJ9x2aG98ha9a4z53rfslFO+fba/dt/uabYafbWXfFWbe9Nc3sBXf/ZNefY93vpv1XgOPd/YLo/XOAI9x9Wqd1/gcoBz4BDAJ+6e7zEmkbSzH01gRYuTKYymnWLJg4Mfn2jY3BLANf+xrcdBMsXw5VVcFNRER29eijcNJJcP/9cPrpYafJPbfeCn//O1x1FZQlMmt3Bv3gB0EHi3/8A0ZG+wbcckswRuhf/wpHHhluvs66660Zbxd6Nz/Huh9zmz08Z8f2xwLHAv2BZ8xsSYJtg42YTQImAYwYMSKBWPlv+HDYujXoFJBKcTZwILzwAvTrF9z/2MfSm09EpNDMng177x0UaLKr888PO0GgtTUYcmrChB2FGcBZZ8Fll8GcOblVnHUnXm/Ng83sAzPbAoyJ/txx/6AEnnsNwZRPHYYDa2Os85i7N7n7BoIeoQcn2BYAd7/Z3SPuHqmurk4gVv4rK4NDDkm9xybARz+6443717/C1VenJZqISMFZtQoWLIALLoDy8rDT5K62tmBey8bG8DI8+ii88w5Mnrzz8oED4Zxz4P/+DzZtCidbMrotzty91N0r3X2Qu5dFf+64n8jb8zlgfzMbZWZ9gDOB+V3WeRj4tJmVmVkFcATweoJti1okAs8/H/wxJOPpp+ErX+Ff4+MAPPkkzJgBzc1pjSgiUhBuuQXMguJMuvfcc3DCCXDPPeFlmD07OLt0wgm7PjZtWjDQb8dZo1yWseFx3X07MA14nKDgutfdXzWzKWY2JbrO68BjwMvA34C57v5Kd20zlTUfRSLQ1BSc40/GTTfBH/4Au+22Y9k++wT/vvNO+vKJiBSKDz6AU0+FIrlyJmVHHAEHHRQUSGHo+E684ILY1719/OPBDDsVFdnPlqyMXrbn7guABV2Wze5yfyYwM5G2ssOnPgVXXJHcm2zduuBi1qlToX//HcuHDQv+XbsWamrSm1NEJN/NmgXt7WGnyH1mwenEadOCy24iu1zmnlkDBkBDA7TEmdO8uTnovHDYYUExmavCn1hKUlJTAz/7Gey3X+Jtbr89uFiy67n4zsWZiIjssHp18G+Jvi0TcvbZwUGDbB89a22FDz+E0tKeD1r84Adw7bXZyZUqvd3yWHMzvPJKYuu2t8PNN8NnPxsc2u1MxZmIyK7efDPoOHX77WEnyR+DBwejCPzlL8lfE90b998fnHZuaIi/Xv/+wTBSDz4I772XnWypUHGWx777XTj8cNi+ved1W1qC/9Fcdtmuj+22W/Am/Y//SH9GEZF8dfPNwZGY448PO0l+mTkzOHBQmsXpK+fMCY6YjRrV87qTJwdH2nK56FZxlscikeDo2euv97xu//7w4x/HHqPHLBiUVoftRUQCLS3Bl/eppwbjm0nihgwJLshvbQ0mtMy05cuDUQcmT07se+xjH4Ojjw6K71y9llBfx3ms42LLnsY7e+cdeOih+EfYbr8drrkmbdFERPLa/ffDxo27XqMriVm6NDjNuHhx5rc1Z04w/tx55yXeZsoU2HdfWL8+c7l6Q8VZHtt/fxg0qOfibO5cOO20YCDF7ixcGIzlIyIicMcdUFsLxx4bdpL89NGPBjPZZLpjQHMz3HlnMKXWnnsm3u6MM+Cpp2Do0Mxl6w0VZ3mspATGjg3+h9Kd7duDQ7fHHRd80HRn2DB1CBAR6XDvvcFNl3ukpmNE/t/+NjgCmSn9+gVHOb/73eTaWXSSyHXrcnPGAL3t8tyPfxz/dOTChbBmTXAIN56994YtW8KddkNEJFcMGQKHHhp2ivw2eXJw7d6dd2ZuG2ZwzDEwZkzybTdsCE69Xn99+nP1loqzPPeZz8BRR3X/+Jw5sNdePU/W2zGchmYJEJFitnUrfO5z8Mc/hp0k/x10UDDJ+Jw5mekYsGwZXHpp6kNiVFUFw0vdcktiox5kk4qzPNfWBg8/HPu6s+ZmeOONxCbrHTYsWGfDhszkFBHJB//3f/CnP8We/keSd9VVwbSBmTB7dnDrzWT0U6YEZ5cWLkxfrnQwz0Y/1yyJRCJe19PV8QXGPRin7MwzY1942d4eHFbuPF1TLG1tweFhXV8hIsXsiCOCSzxefXXHdUmSexobg4MKp54K8+al/jytrcFMO4ccAr/7XZrCJcHMlrr7LhNd6as4z5kFQ2p0rUm3b4dt24Jiq6fCDILBAlWYiUgxe+EF+NvfgqMpKszSZ80a+Na34N130/ecv/lNUET3dD11T8rLg7NLjz8edA7IFfo6LgCRCLz88s6TvT78MAwfHgzOl6hvflPDaYhI8ZozJ/jP7DnnhJ2ksDQ1wXXXpW9EfvfgVOlBB8H48b1/vmnTgqm6khmKI9NUnBWASCQ4NLts2Y5ls2cHU1nsv3/iz/PYY7oIVkSK12c+A9//fnCpiKTPRz8a9KhM14j8LS3BachLLknPEc7q6sSmfcomFWcFoGOmgI7xzt58ExYtgkmTkpvbTGOdiUgxmzgRZswIO0VhmjIFVq6EJ57o/XP16we33Qbnn9/75+qwfn0wqsEDD6TvOXtDxVkBGDkyOH154YXB/Y7JepN946o4E5Fi5B6MxfXPf4adpHCdempw2rC3MwZ88EH8gddTtfvuweVBN96Y/udOhYqzArBiBcz+ZQt7D2mmtKSdG37RzP4jW9i6Nbnn6SjOCqgDr0jKGhpg+tQWhlYGf1dDK5uZPrWFhobw2yhfevOVlbYz5evNnHFqz+0kNX36wFlnwcq/9+612m1wO5+KNHPumel9rUpL4ctfhj//oYXqgcnlywh3L5jb2LFjvdgsWOBeVdHo3yn7uddT462Uej01/p2yn3tVRaMvWJD4c82a5T5qlHtjY+byiuSDjr+rGeU7/13NKO/+7ypbbZQvc/mu7KGdpC6b76VU8+3Rv9EvI7Pb6Qqo8xj1TOgFVTpvxVac1dcHb9rFjAteyi63xYzzqopGr68PO6lI/kjl7ypbbZQvnHzSO7n+WoX5nlBxVoAuvWibzyj/ecw3U8ftyvKZPv3ibWFHFckbqfxdZauN8oWTT3on11+rMN8TKs4K0J6Dtno9NXHfUPXU+NDKpoSe77333L/wBfdHHslwcJEclujfVWV5k7/wQtCmakDibb7xDfdvfMO9sjzxv9/HHvOk21WWN3lzc5Bvt/6ZzffLXyafb0jfHZ9Lg/skn29I38TzXXRR8vkS/dyUniX6N1U9YMfrm83XKt3fpcnorjjT9E15rLSknRbvQxlt3a7TShn9S1rY3tZz348tW6CyEn72M7jiinQmFckfif5d9aOFx54o4bjjkmszbHjwt/j2mnY+JLG/3xtvKuH//T+SatePFjZvKWHgQCi1dloSbJNKvnPOLWHRouTztXmwrRJLvE1HvrVrEvud+pe08JH9S2hqSi5fop+b0rOEv6ushb332bHPs/Vapfu7NBmavqkAVQ1sYRUj466zmhFUDdyW0PMNGhTcNJyGFLNE/66qK7dx3HHJt3nrLXjrLagelPjf76RJJN2uunIbAwdG8yXRJpV8t9+eWr4OybTp2E6iv1PVwG0sX558vkQ/N6VnCX9XDdrx+mbztUr3d2k6qDjLYxPPLuHW8vgTi80tv4iJ5yQ+Eq3GOpNil8rfVbbaKF84+aR3cv21ysn3RKxznfl6K7ZrzjLRw+SYY9yPOipzmUVyXSH2NlS+3reT1OX6a6XemirO0q5jDJgry2d6PTX+IWXR8XpmpjQ2y+WXu595ZmayiuSL++5zr6DRL7fE/65S+VtM9e83W9tSPkmXXH+twnpPqDgrYPX17tMv3uZDK5u8tKTNh1Y2+fSLt+l/fiIpuvHG4NNx4peT+7tK5W8x1b/fbG1L+SRdcv21CuM90V1xpt6aIiJdLFsG998PP/whmIWdRkQKlXprSkKefBIOPBBefz3sJCLhOegg+NGPVJiJSDhUnMlOysqCwuytt8JOIhKO224DHYAXkTCpOJOdDBsW/KvhNKQY/fOfcPHFcOutYScRkWKm4kx2svfewb/vvBNuDpEwzJsH27bBlPhDHomIZJSKM9lJ//4wZIiOnEnxcYfZs+GII+Dgg8NOIyLFrCzsAJJ7TjkFamvDTiGSXU8/DcuXw+23h51ERIqdijPZxR13hJ1AJPtWr4ZRo+CMM8JOIiLFTqc1RUSAc86B+nqoqAg7iYgUOxVnsotrroGqKmhvDzuJSHa8/XZwzVmJPhFFJAdk9KPIzI43s7+bWb2ZXRnj8aPNbLOZvRi9/aDTYyvNbFl0uUYdyqI+fWDjxuAmUuja2+Ezn4Fzzw07iYhIIGPXnJlZKXADcBywBnjOzOa7+2tdVv2zu5/YzdMc4+4bMpVRYus81ll1dbhZRDJt0SJYsQJ+8pOwk4iIBDJ55OxwoN7dV7j7h8A9wCkZ3J6kiQailWIyZ05wGv/008NOIiISyGRxtg/QeRKgNdFlXY03s5fMbKGZfaLTcgeeMLOlZjYpgzmlCxVnUizWroWHH4ZvfAP69g07jYhIIJNDacSaMti73H8eGOnujWZ2AvAQsH/0saPcfa2Z7Qn83syWu/vTu2wkKNwmAYwYMSJt4YvZXnvB2WfDfvuFnUQks+66C9ra4MILw04iIrJDJo+crQH27XR/OLDTsRh3/8DdG6M/LwDKzawqen9t9N91wIMEp0l34e43u3vE3SPVukAqLfr2Db60jj027CQimTV9Ovzxj/CRj4SdRERkh0wWZ88B+5vZKDPrA5wJzO+8gpntZWYW/fnwaJ6NZjbAzAZFlw8AvgC8ksGs0oV7MMegSCHr0weOOSbsFCIiO8vYaU13325m04DHgVLgNnd/1cymRB+fDXwFuMjMtgPNwJnu7mY2FHgwWreVAXe7+2OZyiq7OvXU4Hqc554LO4lIZkyeDGPGwMUXh51ERGRnGZ2+KXqqckGXZbM7/Xw9cH2MdisATT0coj32gKVLw04hkhmrVsEtt8B//mfYSUREdqXxsCWmYcPg3XeDi6VFCs3cuWCmjgAikptUnElMw4YFhdn69WEnEUmv1tagOJswAdTBW0RykYoziUljnUmhmj8/OCo8ZUrYSUREYlNxJjEddBDMmAG77x52kuQ0NMD0qS0MrWymtKSdoZXNTJ/aQkND+G1kZ2G9VpPPbWbMx1o44ID0/04iIumg4kxiqq2Fq67Kr4FoFy6EcWOa6D93Fou3jKbF+7B4y2j6z53FuDFNLFwYXhvZWZiv1bNbR/Olhlkc9Um9ViKSo9y9YG5jx451SZ9Nm9zXrw87RWLq692rKhp9MePcg2HadrotZpxXVTR6fX3228jO9FqJiASAOo9Rz+jImXTr4x+H73437BSJuf6aFi5svZHxLIn5+HiWcEHrTdxwXUvW28jO9FqJiMRnQeFWGCKRiNfV1YUdo2AcemjQMeDRR8NO0rOhlc0s3jKaWlZ0u04DNRw5aBmf+WIFAI8/1MwL23tu88myZTz8RAXHHAPVA5tZ0tRzm6Mql/Hu5orUf6ECluhr1bEPr74arvp+4q/VF0+toKYG7rgpue2IiGSbmS1190jX5RkdhFby27Bh+dNbc0NjX0ayKu46I1jNxsZ+vPZacL9xe2Jtmrb3Y/Pm4P6mrYm12dDYL+HsxSbR16pjH771VnKvVcfrm+x2RERyhU5rSrf23jt/irOqgS2sYmTcdVYzgqpB23j1VXj1VagelFib6sptnHpqktsZqIlJu5PsPrzhhuReq1dfhd/+Vq+ViOQvFWfSrWHDYN062L497CQ9m3h2CbeWxx+4am75RUw8pzTrbWRneq1ERHoQq5dAvt7UWzO9nn3W/YYb3LdtCztJz9QDMH/otRIRCdBNb83QC6p03lScFbdHH3UfYI1+uc30emr8Q8q8nhq/snymV1U0+oIFu7ZZsCD4Ar+yvPdtvlPWfRvZ2a9/7V5Bo19eEs5rFa+NiEi2qDiTpH34ofvLL7u/917YSRLz+OPBO/pLx23zoZVNXlrS5kMrm3z6xdviHh2pr3effnEv2lib96fJT/tS/Dayw09/GrxW5/17ll+rBNuIiGRDd8WZhtKQbr3zTnDd2Y03wkUXhZ2mZ6efDn/+M6xZA337Zm+7jY1QWQk//GFwk/ja2oIZKD7yEVi0KOw0IiLh0VAakrQ994SSkvzosbl2bTCh9WWXZbcwAxg4MBiwd+nS7G43X23ZAkcfDaedFnYSEZHcpOJMulVaCnvtlR/F2d13B0dkJk0KZ/snnADvvx/OtvPNkCFwxx1hpxARyV0qziSuffaBt98OO0XPLr0Uxo8PTpeFYebMcLabb959Nzhd/slPhp1ERCR3aZwziStfZgkoK4Ojjgo7BbS3h50gt910E4wdmx8Fv4hIWFScSVyXXZb7R4XOPx+uuy7cDNu3wwEHwI9+FG6OXNbaCrfcAhMmBEdkRUQkNhVnEtenPw1f/GLYKbq3ahXcfjts2hRujrIy6NcP1Fm4e48+GpzSnDw57CQiIrlNxZnEtWEDLFwIH3wQdpLYbrkFzODCC8NOApFIUJwV0Og0aTV7NgwfHnSeEBGR7qk4k7ieey74Mn3llbCT7Kq1FW69Ncg3YkTYaYLibP36YJw12dn778OzzwZFdJm6IYmIxKWPSYlr2LDg33feCTdHLPPnB73/psSf2zprItFhBOvqYN99w82Sa4YMCYrWtrawk4iI5D4VZxJXR3GWiz0299svuH7p+OPDThIYMybIM3x42ElyS3t7cOp54MCwk4iI5Aed1pS49tgDystzszgbOza4jqm0NOwkgX79gjyHHRZ2ktzy29/CQQfpdK+ISKJUnElcJSWw9965V5w98ggsXx52il21tcHrr6tTQGezZ8PWrTuOwoqISHwqzqRHd98N3/te2Cl2aGmB887LrUwdbr0VDjwQVq4MO0luWL4cnnwymFarRJ82IiIJ0TVn0qNcGHm/s/vvh40bc6cjQGdjxwb/Ll0Ko0aFmyUX3HxzcFr8vPPCTiIikj/0f1np0SuvBAO95oo5c4I5ND/3ubCT7Gr0aOjTR4PRAjQ3BxOcn346DB0adhoRkfyh4kx69Mgj8I1vBNcNhe211+Dpp4Nekbl4mqxv36DXpoqzoKPG9dfD5ZeHnUREJL/k4Neb5JpcGuvshReCMbO+/vWwk3Rv7FjNFADBEcSJE3eM/yYiIolRcSY9yqWxzv7934Misbo67CTdu/BCmDcvGN+rWL36KvzsZ7B5c9hJRETyj4oz6VGuFGcd83v26xdujp6MHQsnn5w746+F4cYb4Yc/1IwAIiKpUHEmPdp77+DfsE9rHndccOQsH/zlL/DUU2GnCEdjI9x1F5xxBuy+e9hpRETyj4bSkB7ttltwrVdNTXgZXngB/va34BqmfHD55cERviefDDtJ9v3mN7BlS24OdSIikg905Ex6ZAaHHAKVldnbZkMDTJ/awtDKZkpL2jn6iGb6l7bw6U9nL0NvRCLBWGf5dt1Z1/0+tLKZ6VNbaGhIvM30Kc3stXsLe+6ZvdwiIoVExZkk5OGHszfW2cKFMG5ME/3nzmLxltG0eB+ebx3NtPZZfPHTTSxcmJ0cvRGJBKf33ngj7CSJi7XfF28ZTf+5sxg3JvZ+j9XmpfbRnLt5FuMPzo/XSkQk57h7xm7A8cDfgXrgyhiPHw1sBl6M3n6QaNtYt7Fjx7pkxle/6v7Rj2Z+O/X17lUVjb6Yce7BaBQ73RYzzqsqGr2+PvNZeuPll4PId90VdpLEpLLfC+W1EhEJC1DnMeqZjB05M7NS4AZgAnAgcJaZHRhj1T+7+yHR238l2VayZNiw7PTWvP6aFi5svZHxLIn5+HiWcEHrTdxwXUvmw/TCxz8O/fvnz2C0qez3QnmtRERyTSZPax4O1Lv7Cnf/ELgHOCULbSUDhg0LLvLesiWz27n71+2c3zo77joXtN7E3Xfl9hgNZWVBYXb11WEnSUyi+33unDYWLAju/3peYbxWIiK5JpPF2T7AW53ur4ku62q8mb1kZgvN7BNJtsXMJplZnZnVrV+/Ph25JYZszRKwobEvI1kVd50RrGZDY44PdgYceGDuj8nWIdH93rS9Hx9+GNzftLVwXisRkVySyeLMYizrOqHN88BIdz8Y+BXwUBJtg4XuN7t7xN0j1bk8bHye6yjO3n47s9upGtjCKkbGXWc1I6gauC2zQdJg5Ur49reJ29MxVyS636srt3Hqqcm1yYfXSkQkl2SyOFsD7Nvp/nBgp6uW3P0Dd2+M/rwAKDezqkTaSnYddRRs3AhHH53Z7Uw8u4Rby+MPkDW3/CImnpP7w+83N8MvfhEMSJvrUtnvhfRaiYjkkkwWZ88B+5vZKDPrA5wJzO+8gpntZWYW/fnwaJ6NibSV7OrbNxjt3WId00yjaZf15ZbyqTzDuJiPP8M45pZfxMXT+2Y2SBoccAAMGJAfnQJS2e+F9FqJiOSSjBVn7r4dmAY8DrwO3Ovur5rZFDPr+O/2V4BXzOwlYBZwZrR3acy2mcoqifnpT+GeezK7jdpamHffAE7st4jLmEkDNbRSRgM1zCifyckVi5h33wBqazObIx1KS+HQQ/OjOOvY718oWcTlCe73jjYnVyxiRnl+v1YiIrnEgmE2CkMkEvG6fPgmzFMHHBAUG5ku0ADuvRf+89stbHm/jQ2N/agauI2J55Ry8fS+efVl/61vwU03BZO2l5eHnSa+ujo47DA49tMtvPJS4vu9oQFuuK6Fu+/K79dKRCTbzGypu0e6LtfcmpKwvffO3uTnZ5wBZ5zR+XRYRXY2nGaRCAwcCGvWwKhRYaeJr7oapk+HH/ygL0OGdCzteb/X1sK11/fl2usTbyMiIt3T9E2SsGwNRPvss/Dee5nfTjb827/BunW5X5gBjBwJ115Lp8JMRETCoOJMEtZRnGXyTHh7O5x9Npx5Zua2kU2lpZnvRJEOTzwBf/pTZl9bERFJjIozSdiwYUHx1NiYuW388Y9QXw8XXJC5bWTb1VfndrHpDpdfHtzyoZAUESl0Ks4kYZdcAlu3wqBBmdvG7Nmwxx7w5S9nbhvZtnEjPPgg/xpZP9c88wwsWwZT4g9ZJiIiWaLiTBJWVpbZIyvvvAMPPwznnZc/0x4lIhIJCrNXXgk7SWyzZwcF91lnhZ1ERERAxZkkYdMm+NrXYNGizDz/n/4UnDadNCkzzx+WSLSTdC6O8rJxYzBsyTnnBL1KRUQkfCrOJGF9+sC8ebB0aWaef+LEoMPB/vtn5vnDUlMT9IDMxeJs+fIg2+TJYScREZEOGudMEjZwIFRWZmY4jba2oGfj0KHpf+6wmQXjtnVMHp9LjjoqGIOtTJ8EIiI5Qx/JkpRMjXX25S8HHQFuvTX9z50L5swJO8GuNm2CwYNVmImI5Bqd1pSkZKI4W70aHnkkmIGgkLlDa2vYKXaYOhXGjtXYZiIiuUbFmSSlthb69u15vWTMnRsUCBdemN7nzSXr1gVHBufODTtJ4L334IEH4JhjNLaZiEiu0QkNScrNN6f3+Vpbg4JlwoRg+qBCVV0dXFOXK50Cbr892PfqCCAikntUnEmoHn00GN8sF6/JSiezYEiNXCjO2tuDIvvoo+FjHws7jYiIdKXTmpKUZ5+FY4+FN95Iz/MdeWQw2faECel5vlwWicCrr0Jzc7g5/vhH+Mc/dNRMRCRXqTiTpLS2Bl/uK1em5/mGDoXp04ujx2AkEgwZ8tJL4eY45hj43e/gtNPCzSEiIrGpOJOkdIzVlY4em7feCvfd1/vnyRdHHAHf+U7QMSBMpaVwwgnp79ghIiLpoeJMktIx3EVvi7OWFpgxA+6+u/eZ8sVee8HVV4c7A8KsWfDd72r4DBGRXKbiTJLSvz/stlvvi7MHH4T164vvuqetW+GFF8LZ9vbtMHMmPP+8hs8QEcllKs4kaZ/6FOy+e++eY/ZsGDUKjjsuPZnyxc9/Hlx71tiY/W0vWBBM1VRsBbGISL4pgsuwJd3mz+9d++XL4amnglN8JUX234NIJBjK4sUXgyI3m2bPDq4ZPPHE7G5XRESSU2RfjZIL1q2Dgw+G884LO0n2jR0b/Jvt8c5WroTHHoPzz4fy8uxuW0REkqPiTJLS0ABfPLqFgaXNlJa0M7SymelTW2hoiN9m+tQWhlYGbb56YjPHHNnCli3Zy50rtm6F3Qa08IMrUt9/qbQ5bHQzH69tKbrTyCIi+UjFmSRs4UIYN6aJQ/48i5faR9PifVi8ZTT9585i3JgmFi7svk3/ubNYvCWxNoWqY1+cv3UWL7Rmbv/FarOkaTSnrJrF6ccX1z4XEclL7l4wt7Fjx7pkRn29e1VFoy9mnHswEsNOt8WM86qKRq+v712bQpWt/ad9LiKSP4A6j1HP6MiZJOT6a1q4sPVGxrMk5uPjWcIFrTdxw3UtvWpTqLK1/7TPRUTyn3kBjUYZiUS8Lhdmli5AQyubWbxlNLWs6HadBmo4pHQZow+rAOCNl5r5W3PPbY6qXMa7myvSnjmXJLr/OvbFf/83/PT7zbzUlvg+r6qCvz2V3HZERCQ8ZrbU3SNdl2soDUnIhsa+jGRV3HVGsJqtbf2orAzuv78tsTYbGvulK2bOSnT/deyLfv1ga1ty+3zQoOS3IyIiuUenNSUhVQNbWMXIuOusZgTVldt4/HF4/PHE21QN3JbOqDkp2X0xfTpUD0pun999t/a5iEghUHEmCZl4dgm3lk+Ju87c8ouYeE5pr9oUqmztP+1zEZECEKuXQL7e1Fszc9RzsHfUW1NERLpCvTWlN2prYd59Azi5YhEzymfSQA2tlNFADTPKZ3JyxSLm3TeA2tretSlU2dp/2uciIvlPvTUlKQ0NcMN1Ldx9VxsbGvtRNXAbE88p5eLpfbv9wk+lTaHK1v7TPhcRyX3d9dZUcSYiIiISgu6KM53WFBEREckhKs5EREREcoiKMxEREZEcktHizMyON7O/m1m9mV0ZZ73DzKzNzL7SadlKM1tmZi+amS4kExERkaKQsembzKwUuAE4DlgDPGdm8939tRjr/Qx4PMbTHOPuGzKVUURERCTXZPLI2eFAvbuvcPcPgXuAU2Ks903gfmBdBrOIiIiI5IVMFmf7AG91ur8muuxfzGwf4DRgdoz2DjxhZkvNbFJ3GzGzSWZWZ2Z169evT0NsERERkfBksjizGMu6Dqr2P8B33L0txrpHufuhwATgYjP7TKyNuPvN7h5x90h1dXWvAouIiIiELWPXnBEcKdu30/3hwNou60SAe8wMoAo4wcy2u/tD7r4WwN3XmdmDBKdJn463waVLl24ws1VJZKwCdE2b9kNn2hc7aF/soH0R0H7YQftiB+2LHZLdFyNjLcxkcfYcsL+ZjQLeBs4EJnZewd1HdfxsZncAj7r7Q2Y2AChx9y3Rn78A/FdPG3T3pA6dmVldrJF5i432ww7aFztoX+ygfRHQfthB+2IH7Ysd0rUvMlacuft2M5tG0AuzFLjN3V81synRx2NdZ9ZhKPBg9IhaGXC3uz+WqawiIiIiuSKTR85w9wXAgi7LYhZl7v71Tj+vAA7OZDYRERGRXFTsMwTcHHaAHKH9sIP2xQ7aFztoXwS0H3bQvthB+2KHtOwLc+/agVJEREREwlLsR85EREREckpRFmeJzvlZDIp5DlMzu83M1pnZK52W7W5mvzezN6P/7hZmxmzpZl/8yMzejr43XjSzE8LMmA1mtq+Z/cnMXjezV83skujyontfxNkXRfW+MLN+ZvY3M3spuh9+HF1ejO+J7vZFUb0nOjOzUjN7wcwejd5Py/ui6E5rRufyfINOc34CZ3Wd87NYmNlKIFKMc5hGBzZuBOa5++josp8Dm9z96mjhvpu7fyfMnNnQzb74EdDo7r8IM1s2mdnewN7u/ryZDQKWAqcCX6fI3hdx9sUZFNH7woJhAwa4e6OZlQN/AS4BTqf43hPd7YvjKaL3RGdm9i2CMVsr3f3EdH2HFOORs0Tn/JQC5+5PA5u6LD4FuDP6850EX0YFr5t9UXTc/R13fz768xbgdYJp54rufRFnXxQVDzRG75ZHb05xvie62xdFycyGA18C5nZanJb3RTEWZz3O+VlkEprDtIgMdfd3IPhyAvYMOU/YppnZy9HTngV/2qYzM9sP+CTwLEX+vuiyL6DI3hfRU1cvAuuA37t70b4nutkXUGTviaj/Aa4A2jstS8v7ohiLs0Tm/CwmCc1hKkXpJqAWOAR4B7gm1DRZZGYDgfuBS939g7DzhCnGvii694W7t7n7IQTTEB5uZqNDjhSabvZF0b0nzOxEYJ27L83E8xdjcZbInJ9Fo/McpkDHHKbF7L3otTYd19ysCzlPaNz9vegHcTtwC0Xy3oheS3M/8L/u/kB0cVG+L2Lti2J9XwC4+/vAkwTXWBXle6JD531RpO+Jo4CTo9dt3wN8zsx+TZreF8VYnP1rzk8z60Mw5+f8kDOFwswGRC/0xXbMYfpK/FYFbz7wtejPXwMeDjFLqDo+YKJOowjeG9ELnm8FXnf3azs9VHTvi+72RbG9L8ys2syGRH/uD3weWE5xvidi7otie08AuPsMdx/u7vsR1BF/dPezSdP7IqPTN+Wi7ub8DDlWWIp6DlMz+w1wNFBlZmuAHwJXA/ea2fnAauCr4SXMnm72xdFmdgjBaf+VwOSw8mXRUcA5wLLodTUA36U43xfd7Yuziux9sTdwZ7Snfwlwr7s/ambPUHzvie72xV1F9p6IJy2fFUU3lIaIiIhILivG05oiIiIiOUvFmYiIiEgOUXEmIiIikkNUnImIiIjkEBVnIiIiIjlExZmISAxm1tjp5xPM7E0zGxFmJhEpDkU3zpmISDLM7FjgV8AX3H112HlEpPCpOBMR6YaZfZpgOpoT3L0h7DwiUhw0CK2ISAxm1gpsAY5295fDziMixUPXnImIxNYKLAbODzuIiBQXFWciIrG1A2cAh5nZd8MOIyLFQ9eciYh0w923mtmJwJ/N7D13vzXsTCJS+FSciYjE4e6bzOx44Gkz2+DuD4edSUQKmzoEiIiIiOQQXXMmIiIikkNUnImIiIjkEBVnIiIiIjlExZmIiIhIDlFxJiIiIpJDVJyJiIiI5BAVZyIiIiI5RMWZiIiISA75/zWXZY0TFW7hAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',\n",
    "         markerfacecolor='red', markersize=10)\n",
    "plt.title('Error Rate vs. K Value')\n",
    "plt.xlabel('K')\n",
    "plt.ylabel('Error Rate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['HTML', 'JavaScript', 'Ruby', 'C++', 'Ruby', 'Python',\n",
       "       'JavaScript', 'Java', 'JavaScript', 'Ruby'], dtype=object)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=10).fit(X_train, y_train)\n",
    "knn_pred = knn.predict(X_train)\n",
    "knn_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "      <th>clf_predicted</th>\n",
       "      <th>rf_predicted</th>\n",
       "      <th>knn_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C++</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C</td>\n",
       "      <td>Java</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>C</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>CSS</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C++</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C++</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>JavaScript</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode lm_predicted clf_predicted  \\\n",
       "26         HTML  Jupyter Notebook      HTML         HTML          HTML   \n",
       "135        HTML              Ruby      HTML         HTML          HTML   \n",
       "77         Ruby            Python      HTML         Ruby          Ruby   \n",
       "84          C++              Java      HTML          C++           C++   \n",
       "1             C              Java      HTML       Python             C   \n",
       "..          ...               ...       ...          ...           ...   \n",
       "16   JavaScript               CSS      HTML   JavaScript    JavaScript   \n",
       "5           C++            Python      HTML          C++           C++   \n",
       "48   JavaScript            Python      HTML   JavaScript    JavaScript   \n",
       "147         C++              Ruby      HTML          C++           C++   \n",
       "36   JavaScript            Python      HTML   JavaScript    JavaScript   \n",
       "\n",
       "    rf_predicted knn_predicted  \n",
       "26          HTML          HTML  \n",
       "135         HTML    JavaScript  \n",
       "77          Ruby          Ruby  \n",
       "84           C++           C++  \n",
       "1           HTML          Ruby  \n",
       "..           ...           ...  \n",
       "16    JavaScript    JavaScript  \n",
       "5            C++           C++  \n",
       "48    JavaScript           C++  \n",
       "147          C++           C++  \n",
       "36    JavaScript    JavaScript  \n",
       "\n",
       "[125 rows x 7 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results['knn_predicted'] = knn_pred\n",
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-Nearest Neighbors Model\n",
      "Accuracy: 64.00%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual            C  C#  C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  \\\n",
      "knn_predicted                                                                 \n",
      "C++               1   0   13    0     0     0           1                 0   \n",
      "HTML              0   1    0    2    11     0           2                 1   \n",
      "Java              0   0    2    0     1    14           0                 0   \n",
      "JavaScript        0   0    2    0     5     0          14                 0   \n",
      "Jupyter Notebook  0   0    0    0     0     0           0                 0   \n",
      "Python            0   0    1    0     0     0           0                 3   \n",
      "Ruby              1   1    1    0     3     2           2                 0   \n",
      "\n",
      "actual            Python  Ruby  Scala  TypeScript  \n",
      "knn_predicted                                      \n",
      "C++                    3     0      0           0  \n",
      "HTML                   1     0      0           0  \n",
      "Java                   0     0      1           0  \n",
      "JavaScript             1     1      0           2  \n",
      "Jupyter Notebook       1     0      0           0  \n",
      "Python                11     1      0           0  \n",
      "Ruby                   2    17      0           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         2\n",
      "              C#       0.00      0.00      0.00         2\n",
      "             C++       0.72      0.68      0.70        19\n",
      "             CSS       0.00      0.00      0.00         2\n",
      "            HTML       0.61      0.55      0.58        20\n",
      "            Java       0.78      0.88      0.82        16\n",
      "      JavaScript       0.56      0.74      0.64        19\n",
      "Jupyter Notebook       0.00      0.00      0.00         4\n",
      "          Python       0.69      0.58      0.63        19\n",
      "            Ruby       0.59      0.89      0.71        19\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         2\n",
      "\n",
      "        accuracy                           0.64       125\n",
      "       macro avg       0.33      0.36      0.34       125\n",
      "    weighted avg       0.59      0.64      0.60       125\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('K-Nearest Neighbors Model')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(train_results.actual, train_results.knn_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(train_results.knn_predicted, train_results.actual))\n",
    "print('---')\n",
    "print(classification_report(train_results.actual, train_results.knn_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation - Checking for Overfitting\n",
    "\n",
    "We will check the following models for overfitting:\n",
    "1) Baseline Random Weighted Choice  \n",
    "2) Baseline Mode  \n",
    "3) Logistic Regression  \n",
    "4) Decision Tree  \n",
    "5) Random Forest  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Random Weighted Choice Performance on Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_results = pd.DataFrame(dict(actual=y_validate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TypeScript', 'Ruby', 'Python', 'Python', 'JavaScript', 'HTML',\n",
       "       'Ruby', 'Ruby', 'C++', 'C++'], dtype='<U16')"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_validate_pred_random = baseline_language_model(train, num_observations = validate.language.shape[0])\n",
    "baseline_validate_pred_random[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Python</td>\n",
       "      <td>TypeScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>Java</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CSS</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual base_random\n",
       "100  Python  TypeScript\n",
       "126    HTML        Ruby\n",
       "183    Java      Python\n",
       "98   Python      Python\n",
       "20      CSS  JavaScript"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_results['base_random'] = baseline_validate_pred_random\n",
    "validate_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Weighted Random Selection - VALIDATE SET\n",
      "Accuracy: 15.62%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual       C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  Ruby  \\\n",
      "base_random                                                                     \n",
      "C#             0    0     0     0           0                 0       0     1   \n",
      "C++            1    0     1     0           1                 0       0     0   \n",
      "CSS            2    0     0     0           0                 0       0     0   \n",
      "HTML           1    0     1     0           1                 0       1     1   \n",
      "Java           0    0     0     0           2                 0       0     1   \n",
      "JavaScript     1    1     1     1           0                 0       0     0   \n",
      "Python         0    0     1     1           0                 1       1     0   \n",
      "Ruby           0    0     1     2           0                 0       2     2   \n",
      "TypeScript     0    0     0     0           1                 0       1     0   \n",
      "\n",
      "actual       Scala  \n",
      "base_random         \n",
      "C#               0  \n",
      "C++              0  \n",
      "CSS              0  \n",
      "HTML             1  \n",
      "Java             0  \n",
      "JavaScript       0  \n",
      "Python           0  \n",
      "Ruby             0  \n",
      "TypeScript       0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "              C#       0.00      0.00      0.00         0\n",
      "             C++       0.33      0.20      0.25         5\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.17      0.20      0.18         5\n",
      "            Java       0.00      0.00      0.00         4\n",
      "      JavaScript       0.00      0.00      0.00         5\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.25      0.20      0.22         5\n",
      "            Ruby       0.29      0.40      0.33         5\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "      TypeScript       0.00      0.00      0.00         0\n",
      "\n",
      "        accuracy                           0.16        32\n",
      "       macro avg       0.09      0.09      0.09        32\n",
      "    weighted avg       0.16      0.16      0.15        32\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Weighted Random Selection - VALIDATE SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(validate_results.actual, validate_results.base_random)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(validate_results.base_random, validate_results.actual))\n",
    "print('---')\n",
    "print(classification_report(validate_results.actual, validate_results.base_random))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Mode Choice Performance on Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Python</td>\n",
       "      <td>TypeScript</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>Java</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CSS</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual base_random base_mode\n",
       "100  Python  TypeScript      HTML\n",
       "126    HTML        Ruby      HTML\n",
       "183    Java      Python      HTML\n",
       "98   Python      Python      HTML\n",
       "20      CSS  JavaScript      HTML"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_results['base_mode'] = train.language.value_counts().idxmax()\n",
    "validate_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Mode Selection - VALIDATE SET\n",
      "Accuracy: 15.62%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual     C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  Ruby  \\\n",
      "base_mode                                                                     \n",
      "HTML         5    1     5     4           5                 1       5     5   \n",
      "\n",
      "actual     Scala  \n",
      "base_mode         \n",
      "HTML           1  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "             C++       0.00      0.00      0.00         5\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.16      1.00      0.27         5\n",
      "            Java       0.00      0.00      0.00         4\n",
      "      JavaScript       0.00      0.00      0.00         5\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.00      0.00      0.00         5\n",
      "            Ruby       0.00      0.00      0.00         5\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.16        32\n",
      "       macro avg       0.02      0.11      0.03        32\n",
      "    weighted avg       0.02      0.16      0.04        32\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Mode Selection - VALIDATE SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(validate_results.actual, validate_results.base_mode)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(validate_results.base_mode, validate_results.actual))\n",
    "print('---')\n",
    "print(classification_report(validate_results.actual, validate_results.base_mode))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Performance on Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Python</td>\n",
       "      <td>TypeScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>Java</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CSS</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual base_random base_mode lm_predicted\n",
       "100  Python  TypeScript      HTML       Python\n",
       "126    HTML        Ruby      HTML   JavaScript\n",
       "183    Java      Python      HTML         Java\n",
       "98   Python      Python      HTML         Ruby\n",
       "20      CSS  JavaScript      HTML   JavaScript"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_results['lm_predicted'] = lm.predict(X_validate)\n",
    "validate_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Model - VALIDATE SET\n",
      "Accuracy: 34.38%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual        C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "lm_predicted                                                               \n",
      "C++             2    0     0     1           0                 0       0   \n",
      "HTML            1    0     1     1           3                 0       1   \n",
      "Java            0    0     0     1           0                 0       0   \n",
      "JavaScript      2    1     2     0           2                 0       1   \n",
      "Python          0    0     0     1           0                 1       1   \n",
      "Ruby            0    0     2     0           0                 0       2   \n",
      "\n",
      "actual        Ruby  Scala  \n",
      "lm_predicted               \n",
      "C++              1      0  \n",
      "HTML             0      0  \n",
      "Java             0      0  \n",
      "JavaScript       0      0  \n",
      "Python           0      1  \n",
      "Ruby             4      0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "             C++       0.50      0.40      0.44         5\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.14      0.20      0.17         5\n",
      "            Java       1.00      0.25      0.40         4\n",
      "      JavaScript       0.25      0.40      0.31         5\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.25      0.20      0.22         5\n",
      "            Ruby       0.50      0.80      0.62         5\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.34        32\n",
      "       macro avg       0.29      0.25      0.24        32\n",
      "    weighted avg       0.38      0.34      0.32        32\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression Model - VALIDATE SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(validate_results.actual, validate_results.lm_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(validate_results.lm_predicted, validate_results.actual))\n",
    "print('---')\n",
    "print(classification_report(validate_results.actual, validate_results.lm_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree Model Performance on Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "      <th>clf_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Python</td>\n",
       "      <td>TypeScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>Java</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Java</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CSS</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual base_random base_mode lm_predicted clf_predicted\n",
       "100  Python  TypeScript      HTML       Python          HTML\n",
       "126    HTML        Ruby      HTML   JavaScript          HTML\n",
       "183    Java      Python      HTML         Java          Java\n",
       "98   Python      Python      HTML         Ruby        Python\n",
       "20      CSS  JavaScript      HTML   JavaScript           C++"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_results['clf_predicted'] = clf.predict(X_validate)\n",
    "validate_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Model - VALIDATE SET\n",
      "Accuracy: 34.38%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual         C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "clf_predicted                                                               \n",
      "C                0    0     0     1           1                 0       0   \n",
      "C++              0    1     1     0           1                 0       0   \n",
      "HTML             3    0     2     0           2                 0       2   \n",
      "Java             1    0     0     2           0                 0       0   \n",
      "JavaScript       1    0     1     1           0                 0       0   \n",
      "Python           0    0     0     0           1                 1       3   \n",
      "Ruby             0    0     1     0           0                 0       0   \n",
      "\n",
      "actual         Ruby  Scala  \n",
      "clf_predicted               \n",
      "C                 0      0  \n",
      "C++               0      0  \n",
      "HTML              1      0  \n",
      "Java              0      0  \n",
      "JavaScript        0      0  \n",
      "Python            0      1  \n",
      "Ruby              4      0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         0\n",
      "             C++       0.00      0.00      0.00         5\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.20      0.40      0.27         5\n",
      "            Java       0.67      0.50      0.57         4\n",
      "      JavaScript       0.00      0.00      0.00         5\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.50      0.60      0.55         5\n",
      "            Ruby       0.80      0.80      0.80         5\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.34        32\n",
      "       macro avg       0.22      0.23      0.22        32\n",
      "    weighted avg       0.32      0.34      0.32        32\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Decision Tree Model - VALIDATE SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(validate_results.actual, validate_results.clf_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(validate_results.clf_predicted, validate_results.actual))\n",
    "print('---')\n",
    "print(classification_report(validate_results.actual, validate_results.clf_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Model Performance on Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "      <th>clf_predicted</th>\n",
       "      <th>rf_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Python</td>\n",
       "      <td>TypeScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>Java</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Java</td>\n",
       "      <td>Java</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>Python</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CSS</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "      <td>C++</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual base_random base_mode lm_predicted clf_predicted rf_predicted\n",
       "100  Python  TypeScript      HTML       Python          HTML          C++\n",
       "126    HTML        Ruby      HTML   JavaScript          HTML         HTML\n",
       "183    Java      Python      HTML         Java          Java         Java\n",
       "98   Python      Python      HTML         Ruby        Python         Ruby\n",
       "20      CSS  JavaScript      HTML   JavaScript           C++   JavaScript"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_results['rf_predicted'] = rf.predict(X_validate)\n",
    "validate_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Model - VALIDATE SET\n",
      "Accuracy: 31.25%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual        C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "rf_predicted                                                               \n",
      "C++             0    0     0     0           0                 0       1   \n",
      "HTML            2    0     2     2           1                 0       1   \n",
      "Java            0    0     0     1           0                 0       1   \n",
      "JavaScript      2    1     0     0           3                 0       0   \n",
      "Python          0    0     2     0           1                 1       1   \n",
      "Ruby            1    0     1     1           0                 0       1   \n",
      "\n",
      "actual        Ruby  Scala  \n",
      "rf_predicted               \n",
      "C++              0      0  \n",
      "HTML             1      0  \n",
      "Java             1      0  \n",
      "JavaScript       0      0  \n",
      "Python           0      1  \n",
      "Ruby             3      0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "             C++       0.00      0.00      0.00         5\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.22      0.40      0.29         5\n",
      "            Java       0.33      0.25      0.29         4\n",
      "      JavaScript       0.50      0.60      0.55         5\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.17      0.20      0.18         5\n",
      "            Ruby       0.43      0.60      0.50         5\n",
      "           Scala       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.31        32\n",
      "       macro avg       0.18      0.23      0.20        32\n",
      "    weighted avg       0.25      0.31      0.27        32\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Random Forest Model - VALIDATE SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(validate_results.actual, validate_results.rf_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(validate_results.rf_predicted, validate_results.actual))\n",
    "print('---')\n",
    "print(classification_report(validate_results.actual, validate_results.rf_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing top Performing Model\n",
    "1) Baseline Random Weighted Choice  \n",
    "2) Baseline Mode  \n",
    "3) Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Random Weighted Choice Performance on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = pd.DataFrame(dict(actual=y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['C++', 'Ruby', 'Jupyter Notebook', 'Python', 'Ruby', 'C++', 'HTML',\n",
       "       'Ruby', 'Java', 'Java'], dtype='<U16')"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_test_pred_random = baseline_language_model(train, num_observations = test.language.shape[0])\n",
    "baseline_test_pred_random[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>Java</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>TypeScript</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>Java</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random\n",
       "177        Java               C++\n",
       "50   TypeScript              Ruby\n",
       "167        Java  Jupyter Notebook\n",
       "129        HTML            Python\n",
       "114        Ruby              Ruby"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results['base_random'] = baseline_test_pred_random\n",
    "test_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Weighted Random Selection - TEST SET\n",
      "Accuracy: 17.86%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual            C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "base_random                                                                    \n",
      "C                   0    0     0     0           1                 0       0   \n",
      "C++                 1    0     0     1           2                 0       1   \n",
      "HTML                0    0     1     1           0                 0       0   \n",
      "Java                1    0     0     0           1                 0       2   \n",
      "JavaScript          0    0     1     0           0                 1       0   \n",
      "Jupyter Notebook    0    0     0     1           0                 0       0   \n",
      "Python              2    0     2     0           0                 0       0   \n",
      "Ruby                0    0     1     1           0                 0       1   \n",
      "TypeScript          0    1     0     0           0                 0       0   \n",
      "\n",
      "actual            Ruby  TypeScript  \n",
      "base_random                         \n",
      "C                    0           0  \n",
      "C++                  0           0  \n",
      "HTML                 0           0  \n",
      "Java                 1           0  \n",
      "JavaScript           0           0  \n",
      "Jupyter Notebook     0           0  \n",
      "Python               0           0  \n",
      "Ruby                 3           1  \n",
      "TypeScript           0           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "               C       0.00      0.00      0.00         0\n",
      "             C++       0.20      0.25      0.22         4\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.50      0.20      0.29         5\n",
      "            Java       0.00      0.00      0.00         4\n",
      "      JavaScript       0.00      0.00      0.00         4\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.00      0.00      0.00         4\n",
      "            Ruby       0.43      0.75      0.55         4\n",
      "      TypeScript       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.18        28\n",
      "       macro avg       0.11      0.12      0.11        28\n",
      "    weighted avg       0.18      0.18      0.16        28\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Weighted Random Selection - TEST SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(test_results.actual, test_results.base_random)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(test_results.base_random, test_results.actual))\n",
    "print('---')\n",
    "print(classification_report(test_results.actual, test_results.base_random))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Mode Performance on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>Java</td>\n",
       "      <td>C++</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>TypeScript</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>Java</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode\n",
       "177        Java               C++      HTML\n",
       "50   TypeScript              Ruby      HTML\n",
       "167        Java  Jupyter Notebook      HTML\n",
       "129        HTML            Python      HTML\n",
       "114        Ruby              Ruby      HTML"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results['base_mode'] = train.language.value_counts().idxmax()\n",
    "test_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline Model: Weighted Random Selection - TEST SET\n",
      "Accuracy: 17.86%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual     C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  Ruby  \\\n",
      "base_mode                                                                     \n",
      "HTML         4    1     5     4           4                 1       4     4   \n",
      "\n",
      "actual     TypeScript  \n",
      "base_mode              \n",
      "HTML                1  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "             C++       0.00      0.00      0.00         4\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.18      1.00      0.30         5\n",
      "            Java       0.00      0.00      0.00         4\n",
      "      JavaScript       0.00      0.00      0.00         4\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.00      0.00      0.00         4\n",
      "            Ruby       0.00      0.00      0.00         4\n",
      "      TypeScript       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.18        28\n",
      "       macro avg       0.02      0.11      0.03        28\n",
      "    weighted avg       0.03      0.18      0.05        28\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Baseline Model: Weighted Random Selection - TEST SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(test_results.actual, test_results.base_mode)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(test_results.base_mode, test_results.actual))\n",
    "print('---')\n",
    "print(classification_report(test_results.actual, test_results.base_mode))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Model Performance on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>base_random</th>\n",
       "      <th>base_mode</th>\n",
       "      <th>lm_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>Java</td>\n",
       "      <td>C++</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>TypeScript</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>C++</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>Java</td>\n",
       "      <td>Jupyter Notebook</td>\n",
       "      <td>HTML</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>HTML</td>\n",
       "      <td>Python</td>\n",
       "      <td>HTML</td>\n",
       "      <td>JavaScript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Ruby</td>\n",
       "      <td>Ruby</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         actual       base_random base_mode lm_predicted\n",
       "177        Java               C++      HTML         HTML\n",
       "50   TypeScript              Ruby      HTML          C++\n",
       "167        Java  Jupyter Notebook      HTML         Java\n",
       "129        HTML            Python      HTML   JavaScript\n",
       "114        Ruby              Ruby      HTML         HTML"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results['lm_predicted'] = lm.predict(X_test)\n",
    "test_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Model - TEST SET\n",
      "Accuracy: 50.00%\n",
      "---\n",
      "Confusion Matrix\n",
      "actual        C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
      "lm_predicted                                                               \n",
      "C++             2    0     0     0           0                 0       0   \n",
      "HTML            0    0     3     1           2                 0       0   \n",
      "Java            0    0     0     3           0                 0       0   \n",
      "JavaScript      0    1     2     0           1                 0       1   \n",
      "Python          2    0     0     0           0                 1       3   \n",
      "Ruby            0    0     0     0           1                 0       0   \n",
      "\n",
      "actual        Ruby  TypeScript  \n",
      "lm_predicted                    \n",
      "C++              0           1  \n",
      "HTML             1           0  \n",
      "Java             0           0  \n",
      "JavaScript       1           0  \n",
      "Python           0           0  \n",
      "Ruby             2           0  \n",
      "---\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "             C++       0.67      0.50      0.57         4\n",
      "             CSS       0.00      0.00      0.00         1\n",
      "            HTML       0.43      0.60      0.50         5\n",
      "            Java       1.00      0.75      0.86         4\n",
      "      JavaScript       0.17      0.25      0.20         4\n",
      "Jupyter Notebook       0.00      0.00      0.00         1\n",
      "          Python       0.50      0.75      0.60         4\n",
      "            Ruby       0.67      0.50      0.57         4\n",
      "      TypeScript       0.00      0.00      0.00         1\n",
      "\n",
      "        accuracy                           0.50        28\n",
      "       macro avg       0.38      0.37      0.37        28\n",
      "    weighted avg       0.51      0.50      0.49        28\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression Model - TEST SET')\n",
    "print('Accuracy: {:.2%}'.format(accuracy_score(test_results.actual, test_results.lm_predicted)))\n",
    "print('---')\n",
    "print('Confusion Matrix')\n",
    "print(pd.crosstab(test_results.lm_predicted, test_results.actual))\n",
    "print('---')\n",
    "print(classification_report(test_results.actual, test_results.lm_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>actual</th>\n",
       "      <th>C++</th>\n",
       "      <th>CSS</th>\n",
       "      <th>HTML</th>\n",
       "      <th>Java</th>\n",
       "      <th>JavaScript</th>\n",
       "      <th>Jupyter Notebook</th>\n",
       "      <th>Python</th>\n",
       "      <th>Ruby</th>\n",
       "      <th>TypeScript</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lm_predicted</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C++</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HTML</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Java</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JavaScript</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Python</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ruby</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "actual        C++  CSS  HTML  Java  JavaScript  Jupyter Notebook  Python  \\\n",
       "lm_predicted                                                               \n",
       "C++             2    0     0     0           0                 0       0   \n",
       "HTML            0    0     3     1           2                 0       0   \n",
       "Java            0    0     0     3           0                 0       0   \n",
       "JavaScript      0    1     2     0           1                 0       1   \n",
       "Python          2    0     0     0           0                 1       3   \n",
       "Ruby            0    0     0     0           1                 0       0   \n",
       "\n",
       "actual        Ruby  TypeScript  \n",
       "lm_predicted                    \n",
       "C++              0           1  \n",
       "HTML             1           0  \n",
       "Java             0           0  \n",
       "JavaScript       1           0  \n",
       "Python           0           0  \n",
       "Ruby             2           0  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(test_results.lm_predicted, test_results.actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}