urls.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract already-prepared urls data\n",
    "\n",
    "Data overview [in this py notebook](https://github.com/sisudata/coloring/blob/master/coloring.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "cd /tmp\n",
    "test -e coloring || git clone https://github.com/sisudata/coloring\n",
    "cd coloring\n",
    "if ! [ -d url_svmlight ] ; then\n",
    "    wget --quiet \"http://www.sysnet.ucsd.edu/projects/url/url_svmlight.tar.gz\"\n",
    "    tar xzf url_svmlight.tar.gz\n",
    "fi\n",
    "\n",
    "if ! [ -f parallelSort.o ] || ! [ -f u4_sort.so ] || ! [ -f u8_sort.so ]; then\n",
    "    ./build.sh 2>/dev/null\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "orig_wd = os.getcwd()\n",
    "os.chdir('/tmp/coloring')\n",
    "import numpy as np\n",
    "\n",
    "import random\n",
    "np.random.seed(1234)\n",
    "random.seed(1234)\n",
    "\n",
    "import utils_graph_coloring as urlutils\n",
    "\n",
    "Xcontinuous, Xcsr, _, y, nrows, ncols = urlutils.get_all_data()\n",
    "\n",
    "os.chdir(orig_wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from svmlight_loader_install import dump_svmlight_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urls-data/cont00.svm\n",
      "urls-data/sps00.svm\n",
      "urls-data/cont01.svm\n",
      "urls-data/sps01.svm\n"
     ]
    }
   ],
   "source": [
    "ixs = np.arange(nrows, dtype=int)\n",
    "# malicious urls is temporally correlated, only 1 split to make\n",
    "cut = int(.7 * nrows)\n",
    "train_ixs, test_ixs = ixs[:cut], ixs[cut:]\n",
    "\n",
    "Xcontinuous_train = Xcontinuous[:cut, :]\n",
    "Xcontinuous_test = Xcontinuous[cut:, :]\n",
    "Xcsr_train = Xcsr[:cut, :]\n",
    "Xcsr_test = Xcsr[cut:, :]\n",
    "y_train = y[:cut]\n",
    "y_test = y[cut:]\n",
    "\n",
    "print('urls-data/cont00.svm')\n",
    "dump_svmlight_file(Xcontinuous_train, y_train, 'urls-data/cont00.svm')\n",
    "print('urls-data/sps00.svm')\n",
    "dump_svmlight_file(Xcsr_train, y_train, 'urls-data/sps00.svm')\n",
    "print('urls-data/cont01.svm')\n",
    "dump_svmlight_file(Xcontinuous_test, y_test, 'urls-data/cont01.svm')\n",
    "print('urls-data/sps01.svm')\n",
    "dump_svmlight_file(Xcsr_test, y_test, 'urls-data/sps01.svm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1677291, 3381345)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xcsr_train.shape # (1677291, 3381345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1677291, 64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xcontinuous_train.shape # (1677291, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}