2_Hyperparameter_search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Hyperparameter optimization\n",
    "\n",
    "This notebook implements the entire processing pipeline to find optimal hyperparameters. It is composed of the following components:\n",
    "1. Text preprocessor that filters and normalizes the input ([`Filter` + `TextPreprocessor`](#Text-Preprocessing))\n",
    "2. An adapter that allows easy iteration over the content of Pandas DataFrames ([`Corpus_adapter`](#Corpus-Adapter))\n",
    "3. The [`Doc2Vec Transformer`](#Doc2Vec-Transformer) builds a document embedding model and transforms text pieces into their Doc2Vec representation.\n",
    "4. The [`Doc2Vec-Classifier`](#Doc2Vec-Classifier) predicts the class/category of a document.\n",
    "5. In the [`Pipeline-and-Hyperparameter-optimization`](#Pipeline-and-Hyperparameter-optimization) section the aforementioned pieces are put together and the processing is carried out.\n",
    "\n",
    "You may have noticed that there is a certain interface design style (e.g. `fit()`, `transform()` and `predict()` methods). This is due to the framework provided by [scikit-learn](http://scikit-learn.org/stable/) which allows a simple automation of the hyperparameter search."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import codecs\n",
    "import unicodedata\n",
    "import re\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "class Filter():\n",
    "    def __init__(self, text=''):\n",
    "        self.s = text\n",
    "        self._lem = WordNetLemmatizer()\n",
    "\n",
    "    def removeHeader(self):\n",
    "        if '\\n\\n' in self.s:\n",
    "            self.s = self.s[self.s.index('\\n\\n'):]\n",
    "        return self\n",
    "            \n",
    "    def normalizeCaracters(self):\n",
    "        # converts 'ueber' to 'uber'\n",
    "        self.s = unicodedata.normalize(\"NFKD\", self.s).encode(\"ascii\", \"ignore\").decode(\"utf8\")\n",
    "        return self\n",
    "    \n",
    "    def selectCharacters(self):\n",
    "        # Lets pass only meaningful characters \n",
    "        self.s = re.sub(r'([^a-zA-Z0-9 \\-\\_%])', '', self.s)\n",
    "        return self\n",
    "    \n",
    "    def multipleDots(self):\n",
    "        # removes multiple dots with optional whitespaces in between\n",
    "        self.s = re.sub(r'((\\.\\s*){2,})', '', self.s)\n",
    "        return self\n",
    "    \n",
    "    def multipleSpaces(self):\n",
    "        # Removes multiple whitespaces\n",
    "        self.s = re.sub(r'(\\s{2,})', ' ', self.s)\n",
    "        return self\n",
    "    \n",
    "    def lower(self):\n",
    "        # lower cases everything\n",
    "        self.s = self.s.lower()\n",
    "        return self\n",
    "    \n",
    "    def shortTokens(self):\n",
    "        # removes tokens shorter than minLen\n",
    "        self.s = re.sub(r'(?<=\\s)[\\w?!%,.;:\\/]{1,3}(?=\\s|\\Z)', '', self.s)\n",
    "        return self\n",
    "        \n",
    "    def digits(self):\n",
    "        # removes all digits except digits that represent years\n",
    "        self.s = re.sub(r'\\b(?!(\\D\\S*|[12][0-9]{3})\\b)\\S+\\b', '', self.s)\n",
    "        return self\n",
    "    \n",
    "    def removeHtml(self):\n",
    "        self.s = re.sub(r'<.*?>', '', self.s)\n",
    "        return self\n",
    "    \n",
    "    def removeMailAddresses(self):\n",
    "        self.s = re.sub(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+', '', self.s)\n",
    "        return self\n",
    "    \n",
    "    def removeQuotes(self):\n",
    "        self.s = re.sub(r'[\"\\']', '', self.s)\n",
    "        return self    \n",
    "    \n",
    "    def replaceNewline(self):\n",
    "        self.s = self.s.replace('\\n', ' ')\n",
    "        return self\n",
    "    \n",
    "    def lemmatize(self):\n",
    "        n = [self._lem.lemmatize(w, 'n') for w in self.s.split()]\n",
    "        v = [self._lem.lemmatize(w, 'v') for w in self.s.split()]\n",
    "        \n",
    "        res = []\n",
    "        for i, w in enumerate(self.s.split()):\n",
    "            if w != n[i]:\n",
    "                res.append(n[i])\n",
    "            elif w != v[i]:\n",
    "                res.append(v[i])\n",
    "            else:\n",
    "                res.append(w)\n",
    "                \n",
    "        self.s = ' '.join(res)\n",
    "        return self\n",
    "    \n",
    "    def result(self):\n",
    "        return self.s\n",
    "    \n",
    "    def __call__(self):\n",
    "        # Calls the default usage of this class.\n",
    "        return self.removeHeader()\\\n",
    "                    .removeHtml() \\\n",
    "                    .removeMailAddresses() \\\n",
    "                    .normalizeCaracters() \\\n",
    "                    .replaceNewline() \\\n",
    "                    .selectCharacters() \\\n",
    "                    .removeQuotes()\\\n",
    "                    .digits() \\\n",
    "                    .multipleDots() \\\n",
    "                    .lower() \\\n",
    "                    .shortTokens() \\\n",
    "                    .multipleSpaces() \\\n",
    "                    .result()\n",
    "\n",
    "\n",
    "class TextPreprocessor():    \n",
    "    \"\"\"\n",
    "    A tiny wrapper around the Filter class, to comply with the sklearn interface\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        res = []\n",
    "        for x in X:\n",
    "            res.append(Filter(x)())\n",
    "        return res\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        return {}\n",
    "    \n",
    "    def set_params(self, **params):\n",
    "        return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Corpus Adapter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The `Corpus_adapter` class wraps around the pandas dataframe. It creates a sub dataset by randomly picking `sample_size` documents. Since this class implements the `__iter__` method, instances allow simple iteration over the sub dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "\n",
    "class Corpus_adapter():\n",
    "    def __init__(self, corpus, sample_size=10000, random_state=42):\n",
    "        self.df = pd.read_pickle(corpus)\n",
    "        \n",
    "        assert sample_size >= 0, 'Sample size must be positive'\n",
    "        if sample_size >= len(self.df):\n",
    "            print('sample_size to large. will be set to max val: {}'.format(len(self.df)))\n",
    "            sample_size = len(self.df)\n",
    "        \n",
    "        rnd = np.random.RandomState(random_state)\n",
    "        self.sample = rnd.choice(self.df.index, sample_size, replace=False)\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for idx in self.sample:\n",
    "            d = self.df.ix[idx]\n",
    "            yield {\n",
    "                    'idx': idx, \n",
    "                    'text': d['text'], \n",
    "                    'category': d['category']\n",
    "                }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Doc2Vec Transformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This transformer class implements the methods necessary to be processed by `sklearn.pipeline.Pipeline` (`fit` and `transform`) as well as `sklearn.model_selection.GridSearchCV` (`get_params` and `set_params`).\n",
    "It contains the Doc2Vec model which is trained by the `fit` method. The `transform`method accepts an array-like object containing text documents that are transformed to their vector representation.\n",
    "\n",
    "An exhaustive description of the hyperparameters can be found [here](https://radimrehurek.com/gensim/models/doc2vec.html#gensim.models.doc2vec.Doc2Vec).\n",
    "\n",
    "__Note__: `set_params` and `get_params` have no effect on the model whatsoever. Their purpose is merely to conform to the interface `sklearn.model_selection.GridSearchCV` defines. Changing a model parameter requires re-training. Fortunately `GridSearchCV` instanciates for each configuration a new `D2VTransformer` and calls the `fit` method, which ensures appropriate behavior."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import gensim\n",
    "import logging\n",
    "from gensim.models import Doc2Vec\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict, OrderedDict\n",
    "\n",
    "class D2VTransformer():\n",
    "    def __init__(self, **kwargs):\n",
    "        self.params = OrderedDict({\n",
    "                'size': 24,\n",
    "                'window': 10,\n",
    "                'workers': 1, \n",
    "                'min_count': 3, \n",
    "                'dm': 0,\n",
    "                'hs': 0,\n",
    "                'iter': 5,\n",
    "                'negative': 10,\n",
    "                'alpha': .025, \n",
    "                'min_alpha': 1e-4, \n",
    "                'batch_words': 1000,\n",
    "                'seed': 42,\n",
    "            })\n",
    "        self.params.update(kwargs)\n",
    "        self.doc_cache = {}\n",
    "        self.model = None\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        class IterList():\n",
    "            def __init__(self, X):\n",
    "                self.X = X\n",
    "            def __iter__(self):\n",
    "                return (TaggedDocument(words=d.split(), tags=[str(i)]) for i, d in enumerate(self.X))\n",
    "        listIter = IterList(X)\n",
    "\n",
    "        # We need to memorize the documents we've already seen (see transform method)\n",
    "        for i, x in enumerate(X):\n",
    "            self.doc_cache[hash(x)] = i\n",
    "        self.model = Doc2Vec(**self.params)\n",
    "        self.model.build_vocab(listIter)\n",
    "\n",
    "        self.model.train(listIter, report_delay=60.0)\n",
    "\n",
    "        self.model.docvecs.init_sims() # this initializes the syn0norm array\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, alpha=1e-1):\n",
    "        res = []\n",
    "        for x in X:\n",
    "            if hash(x) in self.doc_cache: # if seen during training\n",
    "                res.append(self.model.docvecs.doctag_syn0norm[self.doc_cache[hash(x)]])\n",
    "            else:\n",
    "                res.append(self.model.infer_vector(x.split(), alpha=alpha))\n",
    "        return np.array(res)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        return self.params\n",
    "    \n",
    "    def set_params(self, **params):\n",
    "        self.params.update(**params)\n",
    "        return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Doc2Vec Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This class implements a k-nearest neighbors classifier with cosine similarity distance metric (`predict` method). Furthermore, it implements the `score` method which determines the quality of the prediction with respect to a given dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class D2VClassifier():\n",
    "    def __init__(self, **kwargs):\n",
    "        self.params = {\n",
    "            'k': 10\n",
    "        }\n",
    "        self.params.update(kwargs)\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        self.data = X\n",
    "        self.targets = y\n",
    "    \n",
    "    def predict(self, X):\n",
    "        dists = np.dot(X, self.data.transpose())\n",
    "        best = np.fliplr(np.argsort(dists, axis=1))\n",
    "        res = []\n",
    "        for i, bestk in enumerate(best[:, 0:self.params['k']]):\n",
    "            counter = defaultdict(int)\n",
    "            for j, idx in enumerate(bestk):\n",
    "                counter[self.targets[idx]] += dists[i][idx]\n",
    "            counter = [(cat, val) for cat, val in counter.items()]\n",
    "            res.append(sorted(counter, key=lambda x: -x[1])[0][0])\n",
    "        return np.array(res)\n",
    "    \n",
    "    def score(self, X, y):\n",
    "        pred = self.predict(X)\n",
    "        return np.mean(pred == y)\n",
    "\n",
    "    def get_params(self, deep=True):\n",
    "        return self.params\n",
    "    \n",
    "    def set_params(self, **params):\n",
    "        self.params.update(**params)\n",
    "        return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Pipeline and Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "In the following section the previously discussed parts are coming together:\n",
    "1. The dataset gets loaded.\n",
    "2. A transformer-classifier pipeline is built.\n",
    "3. The hyperparameter space gets defined.\n",
    "4. The grid search is carried out\n",
    "\n",
    "Since this part is crucial, a couple of things should be highlighted. The `Pipeline` class integrates multiple transformation steps with a concluding classification step into one object. The resulting object behaves just like a normal scikit-learn classifier with the exception that it accepts an unprocessed corpus. All the preprocessing is hidden within the `Pipeline` object.\n",
    "\n",
    "The hyperpatemeter search (`GridSearchCV`) works on that classifier as well as a set of pre-defined hyperparameters which it is supposed to test for. `GridSearchCV` executes an exhaustive search on the entire parameter space which obviously limits the parameter space that can be searched or rather imposes demands that some machine won't comply with. For situations like these [`RandomizedSearchCV`](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV) is more appropriate.\n",
    "\n",
    "As the `CV` in `GridSearchCV` indicates, Cross-validation is also performed along the way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from itertools import product\n",
    "import multiprocessing as mp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def run_parametersearch(gs_params, corpus_path, jobs=4, sample_sizes=(int(1e4), int(1e3))):\n",
    "    txt_clf = Pipeline([\n",
    "            ('prepro', TextPreprocessor()),\n",
    "            ('trans', D2VTransformer(workers=mp.cpu_count())),\n",
    "            ('cls', D2VClassifier())\n",
    "        ])\n",
    "\n",
    "    res = {}\n",
    "    for sample_size in sample_sizes:\n",
    "        corpus_adapter = Corpus_adapter(corpus=corpus_path, sample_size=sample_size)\n",
    "        gs_clf = GridSearchCV(txt_clf, gs_params, n_jobs=jobs, cv=5, verbose=1, return_train_score=False)\n",
    "\n",
    "        X_train = [f['text'] for f in corpus_adapter]\n",
    "        y_train = [f['category'] for f in corpus_adapter]\n",
    "        res[sample_size] = gs_clf.fit(X_train, y_train)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def gs_to_df(gs_res):\n",
    "    '''\n",
    "    gs_to_df gathers the results of a gridsearch run into a pandas dataframe.\n",
    "    gs_res is a dict who's keys are the sample sizes and who's values are GridSearchCV instances.\n",
    "    '''\n",
    "    df = None\n",
    "    for sample_size, gs in gs_res.items():\n",
    "        cv_report = gs.cv_results_\n",
    "        \n",
    "        mean = cv_report['mean_test_score']\n",
    "        std = cv_report['std_test_score']\n",
    "        params = cv_report['params']\n",
    "        \n",
    "        mean = pd.Series(mean, name='mean')\n",
    "        std = pd.Series(std, name='std')\n",
    "        sample_size = pd.DataFrame([sample_size]*len(params), columns=['sample_size'], dtype=int)\n",
    "        params = pd.DataFrame((pd.Series(p) for p in params))\n",
    "        \n",
    "        params = params.merge(pd.DataFrame(sample_size), left_index=True, right_index=True)\n",
    "        params = params.merge(pd.DataFrame(mean), left_index=True, right_index=True)\n",
    "        params = params.merge(pd.DataFrame(std), left_index=True, right_index=True)\n",
    "        \n",
    "        if df is None:\n",
    "            df = params\n",
    "        else:\n",
    "            df = df.append(params)\n",
    "    df = df.reset_index()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Corpus Configuration\n",
    "The following configuration determines the input paths as well as the result paths for potentially multiple datasets. The input format was already defines in the introduction of one of the previous [notebooks](1.0_Amazon_corpus_to_pandas.ipynb#Amazon-review-corpus).\n",
    "In case you don't define any of the `gs_params` they will be set to one sensible default value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "jobs=4\n",
    "\n",
    "experiment_data = {\n",
    "    'sample_sizes': (1000, 10000),\n",
    "    'corpora': {\n",
    "        'amazon' : {\n",
    "            'input_path': 'data/AMAZON/dataframes/amazon.pkl',\n",
    "        },\n",
    "        '20newsgroups' : {\n",
    "            'input_path': 'data/20NEWSGROUPS/dataframes/20NEWSGROUPS.pkl',\n",
    "        }   \n",
    "    },\n",
    "    'gs_params': { # This dict defines the parameters to test for\n",
    "        'trans__size' : (24,),\n",
    "        'trans__iter': (100,),\n",
    "        'trans__alpha': (.01,),\n",
    "        'trans__negative': (20,),\n",
    "        'trans__window': (5,),\n",
    "        'trans__hs': (1,),\n",
    "        'trans__dm': (0,),\n",
    "        'cls__k': (10,),\n",
    "    },\n",
    "    'param_labels': { # so python names are independent of names in the plots\n",
    "        'trans__size' : r'd',\n",
    "        'trans__iter': r'epoch',\n",
    "        'trans__alpha': r'\\alpha',\n",
    "        'trans__negative': r'ns',\n",
    "        'trans__window': r'win',\n",
    "        'trans__hs': r'hs',\n",
    "        'trans__dm': r'arch',\n",
    "        'cls__k': r'k',        \n",
    "    }\n",
    "}\n",
    "del experiment_data['corpora']['20newsgroups']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import pickle\n",
    "\n",
    "for corpus, config in experiment_data['corpora'].items():\n",
    "    print('Processing {} corpus'.format(corpus))\n",
    "    results = run_parametersearch(\n",
    "        experiment_data['gs_params'], \n",
    "        config['input_path'], \n",
    "        jobs=jobs, \n",
    "        sample_sizes=experiment_data['sample_sizes']\n",
    "    )\n",
    "    df = gs_to_df(results)\n",
    "    experiment_data['corpora'][corpus]['gs_results'] = df \n",
    "\n",
    "# serialize the results of this run\n",
    "now = datetime.now()\n",
    "with open('data/{}_gs_results.pkl'.format(now.strftime(\"%Y%m%d-%H:%M\")), 'wb') as fh:\n",
    "    pickle.dump(experiment_data, fh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Bayesian Optimization\n",
    "The code that is presented in this section is based on the work of [Thomas Huijskens](https://thuijskens.github.io/2016/12/29/bayesian-optimisation/). It implements Bayesian Optimization to find optimal hyperparameter values for the task at hand[1].\n",
    "\n",
    "[1] Snoek, J., Larochelle, H., & Adams, R. P. (2012). Practical bayesian optimization of machine learning algorithms. In Advances in neural information processing systems (pp. 2951-2959)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sklearn.gaussian_process as gp\n",
    "\n",
    "from scipy.stats import norm\n",
    "from scipy.optimize import minimize\n",
    "\n",
    "def expected_improvement(x, gaussian_process, evaluated_loss, greater_is_better=False, n_params=1):\n",
    "    \"\"\" expected_improvement\n",
    "\n",
    "    Expected improvement acquisition function.\n",
    "\n",
    "    Arguments:\n",
    "    ----------\n",
    "        x: array-like, shape = [n_samples, n_hyperparams]\n",
    "            The point for which the expected improvement needs to be computed.\n",
    "        gaussian_process: GaussianProcessRegressor object.\n",
    "            Gaussian process trained on previously evaluated hyperparameters.\n",
    "        evaluated_loss: Numpy array.\n",
    "            Numpy array that contains the values of the loss function for the previously\n",
    "            evaluated hyperparameters.\n",
    "        greater_is_better: Boolean.\n",
    "            Boolean flag that indicates whether the loss function is to be maximised or minimised.\n",
    "        n_params: int.\n",
    "            Dimension of the hyperparameter space.\n",
    "\n",
    "    \"\"\"\n",
    "    x_to_predict = x.reshape(-1, n_params)\n",
    "\n",
    "    mu, sigma = gaussian_process.predict(x_to_predict, return_std=True)\n",
    "\n",
    "    if greater_is_better:\n",
    "        loss_optimum = np.max(evaluated_loss)\n",
    "    else:\n",
    "        loss_optimum = np.min(evaluated_loss)\n",
    "\n",
    "    scaling_factor = (-1) ** (not greater_is_better)\n",
    "\n",
    "    # In case sigma equals zero\n",
    "    with np.errstate(divide='ignore'):\n",
    "        Z = scaling_factor * (mu - loss_optimum) / sigma\n",
    "        expected_improvement = scaling_factor * (mu - loss_optimum) * norm.cdf(Z) + sigma * norm.pdf(Z)\n",
    "        expected_improvement[sigma == 0.0] == 0.0\n",
    "\n",
    "    return expected_improvement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import sklearn.gaussian_process as gp\n",
    "from collections import OrderedDict\n",
    "\n",
    "def gen_params(param_template, size=None):\n",
    "    new_params = OrderedDict({})\n",
    "    for param_name, gen_func in param_template.items():\n",
    "        new_params[param_name] = gen_func(size)\n",
    "    return new_params\n",
    "\n",
    "def param_dict_to_numpy_array(params):\n",
    "    return np.array([[v for k, v in p.items()] for p in params])\n",
    "\n",
    "def bayesian_optimisation(n_iters, sample_loss, param_template, n_pre_samples=5,\n",
    "                          gp_params=None, random_search=10, alpha=1e-5, epsilon=1e-7, verbose=False):\n",
    "    \"\"\" bayesian_optimisation\n",
    "\n",
    "    Uses Gaussian Processes to optimise the loss function `sample_loss`.\n",
    "\n",
    "    Arguments:\n",
    "    ----------\n",
    "        n_iters: integer.\n",
    "            Number of iterations to run the search algorithm.\n",
    "        sample_loss: function.\n",
    "            Function to be optimised.\n",
    "        param_template: dict.\n",
    "            Contains 'param_name': generator_function pairs\n",
    "        x0: array-like, shape = [n_pre_samples, n_params].\n",
    "            Array of initial points to sample the loss function for. If None, randomly\n",
    "            samples from the loss function.\n",
    "        n_pre_samples: integer.\n",
    "            If x0 is None, samples `n_pre_samples` initial points from the loss function.\n",
    "        gp_params: dictionary.\n",
    "            Dictionary of parameters to pass on to the underlying Gaussian Process.\n",
    "        random_search: integer.\n",
    "            Flag that indicates whether to perform random search or L-BFGS-B optimisation\n",
    "            over the acquisition function.\n",
    "        alpha: double.\n",
    "            Variance of the error term of the GP.\n",
    "        epsilon: double.\n",
    "            Precision tolerance for floats.\n",
    "    \"\"\"\n",
    "\n",
    "    x_list = []\n",
    "    y_list = []\n",
    "    \n",
    "    for i in range(n_pre_samples):\n",
    "        new_params = gen_params(param_template)\n",
    "\n",
    "        x_list.append(new_params)\n",
    "        y_list.append(sample_loss(new_params))\n",
    "\n",
    "    # Create the GP\n",
    "    if gp_params is not None:\n",
    "        model = gp.GaussianProcessRegressor(**gp_params)\n",
    "    else:\n",
    "        kernel = gp.kernels.Matern()\n",
    "        model = gp.GaussianProcessRegressor(kernel=kernel,\n",
    "                                            alpha=alpha,\n",
    "                                            n_restarts_optimizer=10,\n",
    "                                            normalize_y=True)\n",
    "    if verbose:\n",
    "        print('beginning optimization')\n",
    "        \n",
    "    for n in range(n_iters):\n",
    "        xp = np.array(param_dict_to_numpy_array(x_list))\n",
    "        yp = np.array(y_list)\n",
    "        model.fit(xp, yp)\n",
    "\n",
    "        # Sample next hyperparameter\n",
    "        new_params = []\n",
    "        for i in range(random_search):\n",
    "            new_params.append(gen_params(param_template))\n",
    "        x_random = param_dict_to_numpy_array(new_params)\n",
    "        \n",
    "        # x_random = np.random.uniform(bounds[:, 0], bounds[:, 1], size=(random_search, n_params))\n",
    "        ei = expected_improvement(x_random, model, yp, greater_is_better=True, n_params=x_random.shape[1])\n",
    "        next_sample = new_params[np.argmax(ei)]\n",
    "\n",
    "        # Duplicates will break the GP. In case of a duplicate, we will randomly sample a next query point.\n",
    "        if np.any(np.abs(param_dict_to_numpy_array([next_sample]) - xp) <= epsilon):\n",
    "            next_sample = gen_params(param_template)\n",
    "\n",
    "        # Sample loss for new set of parameters\n",
    "        cv_score = sample_loss(next_sample)\n",
    "        \n",
    "        if verbose:\n",
    "            print('{} -> {}'.format(next_sample, cv_score))\n",
    "\n",
    "        # Update lists\n",
    "        x_list.append(next_sample)\n",
    "        y_list.append(cv_score)\n",
    "\n",
    "        # Update xp and yp\n",
    "        xp = np.array(x_list)\n",
    "        yp = np.array(y_list)\n",
    "\n",
    "    return xp, yp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def sample_loss():\n",
    "    def inner_loss(params, data=None, target=None):\n",
    "        params = params.copy()\n",
    "        clf_param = {'k': 10}\n",
    "        if 'k' in params:\n",
    "            clf_param['k'] = params.pop('k')\n",
    "            \n",
    "        txt_clf = Pipeline([\n",
    "                ('prepro', TextPreprocessor()),\n",
    "                ('trans', D2VTransformer(workers=mp.cpu_count(), **params)),\n",
    "                ('cls', D2VClassifier(**clf_param))\n",
    "            ])\n",
    "        return cross_val_score(txt_clf, X=data, y=target, cv=5).mean()\n",
    "    \n",
    "    corpus_path = '../data/20NEWSGROUPS/dataframes/20NEWSGROUPS.pkl'\n",
    "    corpus_adapter = Corpus_adapter(corpus=corpus_path, sample_size=18845)\n",
    "    data = [f['text']  for f in corpus_adapter]\n",
    "    target = [f['category'] for f in corpus_adapter]\n",
    "    \n",
    "    return partial(inner_loss, data=data, target=target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "param_template = OrderedDict({\n",
    "    'size': (lambda size: np.random.randint(2, 251, size=size)),\n",
    "    'window': (lambda size: np.random.randint(1, 51, size=size)),\n",
    "    'negative': (lambda size: np.random.randint(1, 21, size=size)),        \n",
    "    'alpha': (lambda size: 10 ** -np.random.uniform(0, 3, size=size)), \n",
    "    'iter': (lambda size: np.random.randint(3, 51, size=size)),        \n",
    "    'dm': (lambda size: np.random.randint(0, 2, size=size)),\n",
    "    'hs': (lambda size: np.random.randint(0, 2, size=size)),\n",
    "    'k': (lambda size: np.random.randint(1, 50, size=size))\n",
    "})\n",
    "\n",
    "xp, yp = bayesian_optimisation(n_iters=100,\n",
    "                               sample_loss=sample_loss(), \n",
    "                               param_template=param_template,\n",
    "                               n_pre_samples=10,\n",
    "                               random_search=10000,\n",
    "                               verbose=True\n",
    "                              )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspecting the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "for ix in list(reversed(yp.argsort()))[:10]:\n",
    "    print(\"Accuracy: {} \\n {}\\n\".format(yp[ix], xp[ix]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}