main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Insight DS, Jun 12, Seda Unal\n",
    "#### Violence Detection in Videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!mkdir test train validation Violence NonViolence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!scp -r Violence NonViolence test/ \n",
    "!scp -r Violence NonViolence train/ \n",
    "!scp -r Violence NonViolence validation/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Run if you have have all the code once, \n",
    "#because then you have distributed the content to other folders \n",
    "!scp ViolenceDataSet/* ViolenceData/. \n",
    "!scp NonViolenceDataSet/* NonViolenceData/. \n",
    "!rm -rf test train validation\n",
    "!mkdir test train validation \n",
    "!scp -r Violence NonViolence test/ \n",
    "!scp -r Violence NonViolence train/ \n",
    "!scp -r Violence NonViolence validation/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!ls NonViolence//"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Split data into Train, Validation and Test Folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "v_files = np.random.permutation(glob(\"ViolenceData/*.avi\"))\n",
    "nv_files = np.random.permutation(glob(\"NonViolenceData/*.avi\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(v_files), len(nv_files), type(v_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_v_idx = int(.8 * len(v_files))\n",
    "train_nv_idx = int(.8 * len(nv_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "int(.8 * train_v_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_v_files = v_files[train_v_idx:]\n",
    "test_nv_files = nv_files[train_nv_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from shutil import move"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for filename in test_v_files:\n",
    "    move(filename, \"test/Violence/\")\n",
    "    \n",
    "for filename in test_nv_files:\n",
    "    move(filename, \"test/NonViolence/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "valid_v_idx = int(.8 * train_v_idx)\n",
    "valid_nv_idx = int(.8 * train_nv_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_v_files, validation_v_files = v_files[:valid_v_idx], v_files[valid_v_idx:train_nv_idx]\n",
    "train_nv_files, validation_nv_files = nv_files[:valid_nv_idx], nv_files[valid_nv_idx:train_nv_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for filename in train_v_files:\n",
    "    move(filename, \"train/Violence/\")\n",
    "\n",
    "for filename in train_nv_files:\n",
    "    move(filename, \"train/NonViolence/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for filename in validation_v_files:\n",
    "    move(filename, \"validation/Violence/\")\n",
    "    \n",
    "for filename in validation_nv_files:\n",
    "    move(filename, \"validation/NonViolence/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create .png images from videos (frame by frame depending on fps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find train/Violence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find train/NonViolence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find validation/Violence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find validation/NonViolence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find test/Violence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!find test/NonViolence/*.avi | while read f; do ffmpeg -i  $f ${f//.avi/-%03d.png}; sleep 5; done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Re-Training of the Inception "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Train and Validation Data Iterators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n",
    "\n",
    "traindatagen = ImageDataGenerator(\n",
    "    # augmentation\n",
    "        rotation_range=45,\n",
    "        width_shift_range=0.2,\n",
    "        height_shift_range=0.2,\n",
    "        shear_range=0.2,\n",
    "        zoom_range=0.2,\n",
    "        horizontal_flip=True,\n",
    "        fill_mode='nearest')\n",
    "\n",
    "valdatagen = ImageDataGenerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 4\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "# output: directory iterator\n",
    "train_generator = traindatagen.flow_from_directory(\n",
    "    directory='train', \n",
    "    shuffle = True,\n",
    "        target_size=(299,299),  # all images will be resized to 299x299\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical')  \n",
    "# since we use binary_crossentropy loss, we need binary labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_generator = valdatagen.flow_from_directory(\n",
    "    directory='validation', \n",
    "        shuffle = False,\n",
    "        target_size=(299,299),  # all images will be resized to 299x299\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical')  # since we use binary_crossentropy loss, we need binary labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications.inception_v3 import InceptionV3\n",
    "from keras.preprocessing import image\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, GlobalAveragePooling2D\n",
    "from keras import backend as K\n",
    "from keras import optimizers\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create the base pre-trained model\n",
    "base_model = InceptionV3(weights='imagenet', include_top=False)\n",
    "# whether to include the fully-connected layer at the top of the network.\n",
    "\n",
    "# add a global spatial average pooling layer\n",
    "x = base_model.output\n",
    "x = GlobalAveragePooling2D()(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's add a fully-connected layer\n",
    "# first arg is the dim of output\n",
    "x = Dense(16, activation='relu')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# and a logistic layer -- let's say we have 2 classes\n",
    "predictions = Dense(2, activation='softmax')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this is the model we will train\n",
    "model = Model(inputs=base_model.input, outputs=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# first: train only the top layers (which were randomly initialized)\n",
    "# i.e. freeze all convolutional InceptionV3 layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=2)\n",
    "\n",
    "nb_epoch = 2\n",
    "\n",
    "## Train the model \n",
    "TrainingLogs = model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch=2,\n",
    "    validation_data=test_generator,\n",
    "    validation_steps=2,\n",
    "    epochs=nb_epoch,\n",
    "    callbacks=[early_stopping],\n",
    "    verbose=1,\n",
    "    pickle_safe=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout = valdatagen.flow_from_directory(directory='test', \n",
    "        target_size=(299, 299),  # all images will be resized to 299x299\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical') \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "model = load_model('EarlyStopped_batch32_Epoch100_Steps230.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "import itertools \n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "\n",
    "\n",
    "# real labels\n",
    "itr2 = valdatagen.flow_from_directory('test', batch_size=1, target_size=(32,32))\n",
    "real_vals = (itr2.classes)\n",
    "n_classes = len(itr2.class_indices)\n",
    "\n",
    "# predicted labels\n",
    "res = model.predict_generator(heldout, steps=1, max_q_size=1, workers=1, pickle_safe=False, verbose=0)\n",
    "predicted_vals = list()\n",
    "for tup in range(len(res)):\n",
    "    predicted_vals.append(0 if res[tup][0]>res[tup][1] else 1)\n",
    "pred_vals = np.array(predicted_vals)  \n",
    "\n",
    "# TP\n",
    "clas1 = pred_vals.sum()\n",
    "clas2 = len(pred_vals)-clas1\n",
    "clas1r = real_vals.sum()\n",
    "clas2r = len(real_vals)-clas1r\n",
    "#abs(pred_vals == real_vals).sum()\n",
    "\n",
    "# Compute Precision-Recall and plot curve\n",
    "precision = dict()\n",
    "recall = dict()\n",
    "average_precision = dict()\n",
    "\n",
    "def ytest(realy):\n",
    "    y_test = list()\n",
    "    y1 = np.array([1,0])\n",
    "    y2 = np.array([0,1])\n",
    "    for i in range(len(real_vals)):\n",
    "        if real_vals[i] == 0:\n",
    "            y_test.append(y1)\n",
    "        else:\n",
    "            y_test.append(y2)\n",
    "    return np.array(y_test)\n",
    "y_test = ytest(real_vals)\n",
    "\n",
    "\n",
    "for i in range(n_classes):\n",
    "    precision[i], recall[i], _ = precision_recall_curve(y_test[:, i],\n",
    "                                                        res[:, i])\n",
    "    average_precision[i] = average_precision_score(y_test[:, i], res[:, i])\n",
    "    \n",
    "    \n",
    "# Compute micro-average ROC curve and ROC area\n",
    "precision[\"micro\"], recall[\"micro\"], _ = precision_recall_curve(y_test.ravel(),\n",
    "    res.ravel())\n",
    "average_precision[\"micro\"] = average_precision_score(y_test, res,\n",
    "                                                     average=\"micro\")\n",
    "\n",
    "\n",
    "# Plot Precision-Recall curve\n",
    "plt.clf()\n",
    "plt.plot(recall[0], precision[0],color='navy',\n",
    "         label='Precision-Recall curve')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('Precision-Recall example: AUC={0:0.2f}'.format(average_precision[0]))\n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.show()\n",
    "\n",
    "# Plot Precision-Recall curve for each class\n",
    "plt.clf()\n",
    "plt.plot(recall[\"micro\"], precision[\"micro\"], color='gold',\n",
    "         label='micro-average Precision-recall curve (area = {0:0.2f})'\n",
    "               ''.format(average_precision[\"micro\"]))\n",
    "\n",
    "\n",
    "# Confusion Matrix\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "\n",
    "cnf_matrix = confusion_matrix(real_vals, pred_vals)\n",
    "np.set_printoptions(precision=3)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=itr2.class_indices,\n",
    "                      title='Confusion matrix, without normalization')\n",
    "\n",
    "# Plot normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=itr2.class_indices, normalize=True,\n",
    "                      title='Normalized confusion matrix')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "itr = valdatagen.flow_from_directory('test', batch_size=1, target_size=(32,32))\n",
    "real_vals = itr.classes\n",
    "\n",
    "res = model.predict_generator(heldout, steps=1, max_q_size=1, workers=1, pickle_safe=False, verbose=0)\n",
    "\n",
    "predicted_vals = list()\n",
    "for tup in range(len(res)):\n",
    "    predicted_vals.append(0 if res[tup][0]>res[tup][1] else 1)\n",
    "    \n",
    "pred_vals = np.array(predicted_vals)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def precision(y_true, y_pred):\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "clas1 = pred_vals.sum()\n",
    "clas2 = len(pred_vals)-clas1\n",
    "clas1r = real_vals.sum()\n",
    "clas2r = len(real_vals)-clas1r\n",
    "TP = abs(pred_vals == real_vals).sum()\n",
    "print(TP)\n",
    "\n",
    "\n",
    "cnf_matrix = confusion_matrix(real_vals, pred_vals)\n",
    "np.set_printoptions(precision=3)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=itr2.class_indices,\n",
    "                      title='Confusion matrix, without normalization')\n",
    "\n",
    "# Plot normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=itr2.class_indices, normalize=True,\n",
    "                      title='Normalized confusion matrix')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}