dataset_initing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "with open(\"proccesed_dataset_new.pickle\", \"rb\") as f:\n",
    "    dataset = pickle.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "shuffle(dataset)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "22427"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### КАСТОМНАЯ ОБРАБОТКА, версия 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "VALID_COUNT = 2427"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "train_dataset = dataset[:len(dataset) - VALID_COUNT]\n",
    "valid_dataset = dataset[len(dataset) - VALID_COUNT:]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open(\"dataset_train_new.pkl\", \"wb\") as f:\n",
    "    pickle.dump(train_dataset, f)\n",
    "\n",
    "with open(\"dataset_valid_new.pkl\", \"wb\") as f:\n",
    "    pickle.dump(valid_dataset, f)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "def gen_text_dataset_file(dataset):\n",
    "    string_to_write = \"\"\n",
    "\n",
    "    for item in tqdm(dataset):\n",
    "        history_task = item[0]\n",
    "        history_correctness = item[1]\n",
    "\n",
    "        assert  len(history_task) == len(history_correctness)\n",
    "\n",
    "        string_to_write += str(len(history_task)) + '\\n'\n",
    "        string_to_write += \",\".join(map(str, history_task)) + '\\n'\n",
    "        string_to_write += \",\".join(map(str, map(int, history_correctness))) + '\\n'\n",
    "    return string_to_write"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20000/20000 [00:55<00:00, 361.64it/s]\n"
     ]
    }
   ],
   "source": [
    "f = open('text_dataset_train_new.txt', 'w')\n",
    "f.write(gen_text_dataset_file(train_dataset))\n",
    "f.close()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2427/2427 [00:01<00:00, 1973.79it/s]\n"
     ]
    }
   ],
   "source": [
    "f = open('text_dataset_valid_new.txt', 'w')\n",
    "f.write(gen_text_dataset_file(valid_dataset))\n",
    "f.close()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### КАСТОМНАЯ ОБРАБОТКА, версия 2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "22427"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "user_ids = []\n",
    "log_ids = []\n",
    "tasks_ids = []\n",
    "correct_ids = []"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22427/22427 [00:04<00:00, 5445.04it/s]\n"
     ]
    }
   ],
   "source": [
    "log_id = 0\n",
    "\n",
    "for user_id in tqdm(range(len(dataset))):\n",
    "    tasks, correct = dataset[user_id]\n",
    "\n",
    "    for task_i in range(len(tasks)):\n",
    "        log_id += 1\n",
    "\n",
    "        user_ids.append(user_id)\n",
    "        log_ids.append(log_id)\n",
    "        tasks_ids.append(tasks[task_i])\n",
    "        correct_ids.append(int(correct[task_i]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "22427"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "new_df = pd.DataFrame({\n",
    "    \"user_id\": user_ids,\n",
    "    \"log_id\": log_ids,\n",
    "    \"sequence_id\": tasks_ids,\n",
    "    \"correct\": correct_ids\n",
    "})"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "data": {
      "text/plain": "78323.25"
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(user_ids) / 100"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "         user_id   log_id  sequence_id  correct\n0              0        1         1822        1\n1              0        2         1841        0\n2              0        3         1860        0\n3              0        4         1879        0\n4              0        5         1898        1\n...          ...      ...          ...      ...\n7832320    22426  7832321          825        0\n7832321    22426  7832322         1939        1\n7832322    22426  7832323         1668        1\n7832323    22426  7832324         2377        1\n7832324    22426  7832325         1011        0\n\n[7832325 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>log_id</th>\n      <th>sequence_id</th>\n      <th>correct</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1822</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>2</td>\n      <td>1841</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>3</td>\n      <td>1860</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>4</td>\n      <td>1879</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>5</td>\n      <td>1898</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7832320</th>\n      <td>22426</td>\n      <td>7832321</td>\n      <td>825</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>7832321</th>\n      <td>22426</td>\n      <td>7832322</td>\n      <td>1939</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7832322</th>\n      <td>22426</td>\n      <td>7832323</td>\n      <td>1668</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7832323</th>\n      <td>22426</td>\n      <td>7832324</td>\n      <td>2377</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7832324</th>\n      <td>22426</td>\n      <td>7832325</td>\n      <td>1011</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>7832325 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [],
   "source": [
    "new_df.to_csv(\"new_repo_dataset_2015.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}