ids_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ids_data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOJnIQ1f05cyAlziQkCC+IT",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/r-dube/CICIDS/blob/main/ids_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nE5yrJmDpFg5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1f3c0a33-f394-48db-bc00-8c72ac5e3625"
      },
      "source": [
        "# Assumes that data is available on locally mounted drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k4p_ke1ZktTU"
      },
      "source": [
        "\"\"\"\n",
        "Data Cleaning and Utility functions for CICIDS 2017 data\n",
        "\"\"\"\n",
        "\n",
        "# Load the top modules that are used in multiple places\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IFYAnMLOo8R6"
      },
      "source": [
        "# Some global variables to drive the script\n",
        "# The indir should match the location of the data\n",
        "# The outdir should be the desired location of the output\n",
        "# indir = '/content/drive/My Drive/Public/MachineLearningCVE/raw/'\n",
        "# outdir = '/content/drive/My Drive/Public/MachineLearningCVE/processed/'\n",
        "# combined_data='cicids2017.csv'\n",
        "# balanced_data='bal-cicids2017.csv'\n",
        "\n",
        "# Uncomment for testing the process of combining data\n",
        "# Balancing is not tested\n",
        "indir = '/content/drive/My Drive/Data/MachineLearningCVE/test/'\n",
        "outdir = '/content/drive/My Drive/Data/MachineLearningCVE/processed/'\n",
        "combined_data='test-cicids2017.csv'"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sxd1RFeRnSQb"
      },
      "source": [
        "# Column name mapping from original data to compact form\n",
        "# All the X** are features and the YY is the label\n",
        "feature_map = {\n",
        " ' Destination Port' : 'X1',\n",
        " ' Flow Duration' : 'X2', \n",
        " ' Total Fwd Packets' : 'X3', \n",
        " ' Total Backward Packets' : 'X4', \n",
        " 'Total Length of Fwd Packets' : 'X5', \n",
        " ' Total Length of Bwd Packets' : 'X6', \n",
        " ' Fwd Packet Length Max' : 'X7', \n",
        " ' Fwd Packet Length Min' : 'X8', \n",
        " ' Fwd Packet Length Mean' : 'X9', \n",
        " ' Fwd Packet Length Std' : 'X10', \n",
        " 'Bwd Packet Length Max' : 'X11', \n",
        " ' Bwd Packet Length Min' : 'X12', \n",
        " ' Bwd Packet Length Mean' : 'X13', \n",
        " ' Bwd Packet Length Std' : 'X14', \n",
        " 'Flow Bytes/s' : 'X15', \n",
        " ' Flow Packets/s' : 'X16', \n",
        " ' Flow IAT Mean' : 'X17', \n",
        " ' Flow IAT Std' : 'X18', \n",
        " ' Flow IAT Max' : 'X19', \n",
        " ' Flow IAT Min' : 'X20', \n",
        " 'Fwd IAT Total' : 'X21', \n",
        " ' Fwd IAT Mean' : 'X22', \n",
        " ' Fwd IAT Std' : 'X23', \n",
        " ' Fwd IAT Max' : 'X24', \n",
        " ' Fwd IAT Min' : 'X25', \n",
        " 'Bwd IAT Total' : 'X26', \n",
        " ' Bwd IAT Mean' : 'X27', \n",
        " ' Bwd IAT Std' : 'X28', \n",
        " ' Bwd IAT Max' : 'X29', \n",
        " ' Bwd IAT Min' : 'X30', \n",
        " 'Fwd PSH Flags' : 'X31', \n",
        " ' Bwd PSH Flags' : 'X32', \n",
        " ' Fwd URG Flags' : 'X33', \n",
        " ' Bwd URG Flags' : 'X34', \n",
        " ' Fwd Header Length' : 'X35', \n",
        " ' Bwd Header Length' : 'X36', \n",
        " 'Fwd Packets/s' : 'X37', \n",
        " ' Bwd Packets/s' : 'X38', \n",
        " ' Min Packet Length' : 'X39', \n",
        " ' Max Packet Length' : 'X40', \n",
        " ' Packet Length Mean' : 'X41', \n",
        " ' Packet Length Std' : 'X42', \n",
        " ' Packet Length Variance' : 'X43', \n",
        " 'FIN Flag Count' : 'X44', \n",
        " ' SYN Flag Count' : 'X45', \n",
        " ' RST Flag Count' : 'X46', \n",
        " ' PSH Flag Count' : 'X47', \n",
        " ' ACK Flag Count' : 'X48', \n",
        " ' URG Flag Count' : 'X49', \n",
        " ' CWE Flag Count' : 'X50', \n",
        " ' ECE Flag Count' : 'X51', \n",
        " ' Down/Up Ratio' : 'X52', \n",
        " ' Average Packet Size' : 'X53', \n",
        " ' Avg Fwd Segment Size' : 'X54', \n",
        " ' Avg Bwd Segment Size' : 'X55', \n",
        " ' Fwd Header Length.1' : 'X56', \n",
        " 'Fwd Avg Bytes/Bulk' : 'X57', \n",
        " ' Fwd Avg Packets/Bulk' : 'X58', \n",
        " ' Fwd Avg Bulk Rate' : 'X59', \n",
        " ' Bwd Avg Bytes/Bulk' : 'X60', \n",
        " ' Bwd Avg Packets/Bulk' : 'X61', \n",
        " 'Bwd Avg Bulk Rate' : 'X62', \n",
        " 'Subflow Fwd Packets' : 'X63', \n",
        " ' Subflow Fwd Bytes' : 'X64', \n",
        " ' Subflow Bwd Packets' : 'X65', \n",
        " ' Subflow Bwd Bytes' : 'X66', \n",
        " 'Init_Win_bytes_forward' : 'X67', \n",
        " ' Init_Win_bytes_backward' : 'X68', \n",
        " ' act_data_pkt_fwd' : 'X69', \n",
        " ' min_seg_size_forward' : 'X70', \n",
        " 'Active Mean' : 'X71', \n",
        " ' Active Std' : 'X72', \n",
        " ' Active Max' : 'X73', \n",
        " ' Active Min' : 'X74', \n",
        " 'Idle Mean' : 'X75', \n",
        " ' Idle Std' : 'X76', \n",
        " ' Idle Max' : 'X77', \n",
        " ' Idle Min' : 'X78', \n",
        " ' Label': 'YY'\n",
        "}\n",
        "\n",
        "# label names (YY) in the data and their\n",
        "# mapping to numerical values\n",
        "label_map = {\n",
        " 'BENIGN' : 0,\n",
        " 'FTP-Patator' : 1,\n",
        " 'SSH-Patator' : 2,\n",
        " 'DoS slowloris' : 3,\n",
        " 'DoS Slowhttptest': 4,\n",
        " 'DoS Hulk' : 5,\n",
        " 'DoS GoldenEye' : 6,\n",
        " 'Heartbleed' : 7,\n",
        " 'Web Attack � Brute Force' : 8,\n",
        " 'Web Attack � XSS' : 8,\n",
        " 'Web Attack � Sql Injection' : 8,\n",
        " 'Infiltration' : 9,\n",
        " 'Bot' : 10,\n",
        " 'PortScan' : 11,\n",
        " 'DDoS' : 12,\n",
        "}\n",
        "\n",
        "num_ids_features = 76\n",
        "num_ids_classes = 13\n",
        "ids_classes = [ 'BENIGN', 'FTP-Patator', 'SSH-Patator', 'DoS slowloris', 'DoS Slowhttptest', 'DoS Hulk', 'DoS GoldenEye', 'Heartbleed', 'Brute Force', 'XSS', 'Sql Injection', 'Infiltration', 'Bot', 'PortScan', 'DDoS',]\n",
        "\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WpL5jkxYnh1O"
      },
      "source": [
        "def ids_combine():\n",
        "    \"\"\"\n",
        "    Combine all csv files to produce a single csv file \n",
        "    Input:\n",
        "        None\n",
        "    Returns:\n",
        "        None\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    import os\n",
        "    import glob\n",
        "    os.chdir(indir)\n",
        "    extension = 'csv'\n",
        "    all_filenames = [i for i in glob.glob('*.{}'.format(extension))]\n",
        "\n",
        "    # combine all files in the list\n",
        "    df = pd.concat([pd.read_csv(f) for f in all_filenames ])\n",
        "\n",
        "    # Drop columns 14 and 15 that have Nan and Infinity in them\n",
        "    df.rename(columns = feature_map, inplace=True)\n",
        "    df.drop(columns=['X15', 'X16'], inplace=True)\n",
        "\n",
        "    # Convert string labels to numeric\n",
        "    df['YY'].replace(label_map, inplace=True)\n",
        "\n",
        "    # export to csv\n",
        "    df.to_csv(outdir + combined_data, index=False)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "US48PaaNnmJv"
      },
      "source": [
        "def ids_balance():\n",
        "    \"\"\"\n",
        "    Balance dataset using a heuristic\n",
        "    Input:\n",
        "        None\n",
        "    Returns:\n",
        "        None\n",
        "    \"\"\"\n",
        "\n",
        "    from sklearn.utils import resample\n",
        "    n = 8000\n",
        "\n",
        "    df = pd.read_csv(outdir + combined_data, delimiter=',')\n",
        "    df0 = df[df.YY == 0]\n",
        "    df1 = df[df.YY == 1]\n",
        "    df2 = df[df.YY == 2]\n",
        "    df3 = df[df.YY == 3]\n",
        "    df4 = df[df.YY == 4]\n",
        "    df5 = df[df.YY == 5]\n",
        "    df6 = df[df.YY == 6]\n",
        "    df7 = df[df.YY == 7]\n",
        "    df8 = df[df.YY == 8]\n",
        "    df9 = df[df.YY == 9]\n",
        "    df10 = df[df.YY == 10]\n",
        "    df11 = df[df.YY == 11]\n",
        "    df12 = df[df.YY == 12]\n",
        "    \n",
        "    df0 = resample(df0, replace=False, n_samples=5*n, random_state=123)\n",
        "    df1 = resample(df1, replace=True, n_samples=n, random_state=123)\n",
        "    df2 = resample(df2, replace=True, n_samples=n, random_state=123)\n",
        "    df3 = resample(df3, replace=True, n_samples=n, random_state=123)\n",
        "    df4 = resample(df4, replace=True, n_samples=n, random_state=123)\n",
        "    df5 = resample(df5, replace=False, n_samples=n, random_state=123)\n",
        "    df6 = resample(df6, replace=False, n_samples=n, random_state=123)\n",
        "    df7 = resample(df7, replace=True, n_samples=n, random_state=123)\n",
        "    df8 = resample(df8, replace=True, n_samples=n, random_state=123)\n",
        "    df9 = resample(df9, replace=True, n_samples=n, random_state=123)\n",
        "    df10 = resample(df10, replace=True, n_samples=n, random_state=123)\n",
        "    df11 = resample(df11, replace=False, n_samples=n, random_state=123)\n",
        "    df12 = resample(df12, replace=False, n_samples=n, random_state=123)\n",
        "\n",
        "    df_sampled = pd.concat([df0, df1, df2, df3, df4, df5, df6, df7, df8, df9, df10, df11, df12])\n",
        "\n",
        "    print (df_sampled.YY.value_counts())\n",
        "\n",
        "    # export to csv\n",
        "    df_sampled.to_csv(outdir + balanced_data, index=False)\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6eN6b5F-vfoc"
      },
      "source": [
        "# def main():\n",
        "# ids_check_version()\n",
        "pd.set_option('display.max_rows', 500)\n",
        "pd.set_option('display.max_columns', 500)\n",
        "\n",
        "# uncomment to clean and combine data files\n",
        "ids_combine()\n",
        "\n",
        "# uncomment to create a class-balanaced version of the data\n",
        "# only works for raw (not test) data\n",
        "# ids_balance ()"
      ],
      "execution_count": 7,
      "outputs": []
    }
  ]
}