sessions/15_news20.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Text classification with 1-D Convolutional Neural Networks\n",
    "\n",
    "In this notebook we are going to use the [GloVe](http://nlp.stanford.edu/projects/glove/) pre-trained embeddings to train a text classifier on the [20 Newsgroup dataset](http://qwone.com/~jason/20Newsgroups/) by using convolutional neural networks. [This project](https://github.com/jarfo/dlsl/blob/master/news20/pretrained_word_embeddings.py) is part of the UPC deep learning for speech and language (dlsl) semminar.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "from utils import plot_curves\n",
    "np.random.seed(1337)\n",
    "\n",
    "from keras.preprocessing.text import text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, Dropout\n",
    "from keras.models import Model\n",
    "import timeit\n",
    "import sys\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "BASE_DIR = './'\n",
    "GLOVE_DIR = BASE_DIR + '../../data/glove/'\n",
    "TRAIN_TEXT_DATA_DIR = BASE_DIR + '../../data/20news/20news-bydate-train/'\n",
    "TEST_TEXT_DATA_DIR = BASE_DIR + '../../data/20news/20news-bydate-test/'\n",
    "MAX_SEQUENCE_LENGTH = 1000\n",
    "EMBEDDING_DIM = 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# utility to read the pre-trained glove vectors\n",
    "def read_glove_vectors(filename):\n",
    "    embeddings_index = {}\n",
    "    f = open(filename)\n",
    "    coefs = None\n",
    "    for i, line in enumerate(f):\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        if coefs is None:\n",
    "            coefs = [[0] * len(values[1:])]\n",
    "        coefs.append(values[1:])\n",
    "        embeddings_index[word] = i + 1\n",
    "    f.close()\n",
    "    coefsm = np.asarray(coefs, dtype='float32')\n",
    "    return coefsm, embeddings_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Converts a list of texts to a matrix of word indices\n",
    "def test_to_sequence(texts, index, max_sequence_length):\n",
    "    texts = map(text_to_word_sequence, texts)\n",
    "    matrix = np.array(pad_sequences([[index[word] for word in text if word in index] for text in texts], max_sequence_length))\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# first, build index mapping words to rows the embeddings matrix\n",
    "print('Reading word vectors.')\n",
    "embedding_matrix, embeddings_index = read_glove_vectors(os.path.join(GLOVE_DIR, 'glove.6B.%dd.txt' % EMBEDDING_DIM))\n",
    "print('Found %s word vectors.' % len(embeddings_index))\n",
    "\n",
    "# second, prepare text samples and their labels\n",
    "print('Processing text dataset')\n",
    "\n",
    "def prepare_texts(text_data_dir, labels_index = {}):\n",
    "    texts = []  # list of text samples\n",
    "    labels = []  # list of label ids\n",
    "    for name in sorted(os.listdir(text_data_dir)):\n",
    "        path = os.path.join(text_data_dir, name)\n",
    "        if os.path.isdir(path):\n",
    "            label_id = labels_index.get(name)\n",
    "            if label_id is None:\n",
    "                label_id = len(labels_index)\n",
    "                labels_index[name] = label_id\n",
    "            for fname in sorted(os.listdir(path)):\n",
    "                if fname.isdigit():\n",
    "                    fpath = os.path.join(path, fname)\n",
    "                    if sys.version_info < (3,):\n",
    "                        f = open(fpath)\n",
    "                    else:\n",
    "                        f = open(fpath, encoding='latin-1')\n",
    "                    texts.append(f.read())\n",
    "                    f.close()\n",
    "                    labels.append(label_id)\n",
    "\n",
    "    return texts, labels, labels_index\n",
    "\n",
    "train_texts, train_labels, labels_index = prepare_texts(TRAIN_TEXT_DATA_DIR)\n",
    "print('Found %s training texts.' % len(train_texts))\n",
    "test_texts, test_labels, labels_index = prepare_texts(TEST_TEXT_DATA_DIR, labels_index)\n",
    "print('Found %s test texts.' % len(test_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# finally, vectorize the text samples into a 2D integer tensor\n",
    "X_train = test_to_sequence(train_texts, embeddings_index, MAX_SEQUENCE_LENGTH)\n",
    "X_val = test_to_sequence(test_texts, embeddings_index, MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "y_train = np.array(train_labels, dtype='int32')\n",
    "y_val = np.array(test_labels, dtype='int32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Exercise:** Define a convolutional architecture (based on 1-D convolutional layers + maxpoolings 1D) to build a classifier that gets GloVe Embeddings injected and has to classify the input context among the possible 20 classes defined in 20news. Advice: Check out the convolutional structures of Keras in https://keras.io/layers/convolutional/ ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# load pre-trained word embeddings into an Embedding layer\n",
    "# note that we set trainable = False so as to keep the embeddings fixed\n",
    "embedding_layer = Embedding(embedding_matrix.shape[0],\n",
    "                            embedding_matrix.shape[1],\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)\n",
    "\n",
    "print('Building model...')\n",
    "beg_t = timeit.default_timer()\n",
    "# TODO: build the conv1D architecture to perform classification on top of the Embeddings input\n",
    "# input is tensor embedding_layer\n",
    "\n",
    "\n",
    "# TODO: compile the model and request the Accuracy metric to be extracted during training\n",
    "\n",
    "end_t = timeit.default_timer()\n",
    "print('Elapsed time to build the model {} s'.format(end_t - beg_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Let's check the summary of the built model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Exercise:** Make a function `num_c1d_parms(num_inputs, num_kernels, kernel_width)` that computes the number of parameters to be adjusted within a C1D layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# TODO: make the num_c1d_params(num_inputs, num_kernels, kernel_width) function and test it with some of your built layers\n",
    "# ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# happy learning!\n",
    "his = model.fit(X_train, y_train[..., np.newaxis],\n",
    "                validation_data=(X_val, y_val[..., np.newaxis]),\n",
    "                nb_epoch=20, batch_size=200, verbose=2)\n",
    "\n",
    "plot_curves(his, 20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}