TransformerTrain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gc\n",
    "import random\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.utils.rnn as rnn_utils\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from pathlib import Path\n",
    "import datatable as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Transformer_torch import TransformerModel\n",
    "from Early_Stopping import EarlyStopping\n",
    "from LRSechduler import NoamOpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types_dict = {\n",
    "    'row_id' : 'int32',\n",
    "    'content_type_id': 'bool',\n",
    "    'timestamp': 'int64',\n",
    "    'user_id': 'int32', \n",
    "    'content_id': 'int16',\n",
    "    'task_container_id' : 'int16',\n",
    "    'answered_correctly': 'int8',\n",
    "    'prior_question_elapsed_time': 'int64'\n",
    "}\n",
    "target = 'answered_correctly'\n",
    "train_df = dt.fread('./data/sample_train.csv', columns=set(data_types_dict.keys())).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10000 entries, 0 to 9999\n",
      "Data columns (total 8 columns):\n",
      " #   Column                       Non-Null Count  Dtype  \n",
      "---  ------                       --------------  -----  \n",
      " 0   row_id                       10000 non-null  int32  \n",
      " 1   timestamp                    10000 non-null  int64  \n",
      " 2   user_id                      10000 non-null  int32  \n",
      " 3   content_id                   10000 non-null  int32  \n",
      " 4   content_type_id              10000 non-null  bool   \n",
      " 5   task_container_id            10000 non-null  int32  \n",
      " 6   answered_correctly           10000 non-null  int32  \n",
      " 7   prior_question_elapsed_time  9742 non-null   float64\n",
      "dtypes: bool(1), float64(1), int32(5), int64(1)\n",
      "memory usage: 361.5 KB\n"
     ]
    }
   ],
   "source": [
    "train_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "      <th>content_id</th>\n",
       "      <th>content_type_id</th>\n",
       "      <th>task_container_id</th>\n",
       "      <th>answered_correctly</th>\n",
       "      <th>prior_question_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>115</td>\n",
       "      <td>5692</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>56943</td>\n",
       "      <td>115</td>\n",
       "      <td>5716</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>118363</td>\n",
       "      <td>115</td>\n",
       "      <td>128</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>55000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>131167</td>\n",
       "      <td>115</td>\n",
       "      <td>7860</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>137965</td>\n",
       "      <td>115</td>\n",
       "      <td>7922</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>9995</td>\n",
       "      <td>2868187305</td>\n",
       "      <td>91216</td>\n",
       "      <td>1124</td>\n",
       "      <td>False</td>\n",
       "      <td>775</td>\n",
       "      <td>0</td>\n",
       "      <td>18000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>9996</td>\n",
       "      <td>2868272689</td>\n",
       "      <td>91216</td>\n",
       "      <td>810</td>\n",
       "      <td>False</td>\n",
       "      <td>776</td>\n",
       "      <td>1</td>\n",
       "      <td>18000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>9997</td>\n",
       "      <td>2868367298</td>\n",
       "      <td>91216</td>\n",
       "      <td>1245</td>\n",
       "      <td>False</td>\n",
       "      <td>777</td>\n",
       "      <td>1</td>\n",
       "      <td>17000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>9998</td>\n",
       "      <td>2868439137</td>\n",
       "      <td>91216</td>\n",
       "      <td>711</td>\n",
       "      <td>False</td>\n",
       "      <td>778</td>\n",
       "      <td>1</td>\n",
       "      <td>17000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9999</td>\n",
       "      <td>2868510836</td>\n",
       "      <td>91216</td>\n",
       "      <td>1123</td>\n",
       "      <td>False</td>\n",
       "      <td>779</td>\n",
       "      <td>1</td>\n",
       "      <td>16000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      row_id   timestamp  user_id  content_id  content_type_id  \\\n",
       "0          0           0      115        5692            False   \n",
       "1          1       56943      115        5716            False   \n",
       "2          2      118363      115         128            False   \n",
       "3          3      131167      115        7860            False   \n",
       "4          4      137965      115        7922            False   \n",
       "...      ...         ...      ...         ...              ...   \n",
       "9995    9995  2868187305    91216        1124            False   \n",
       "9996    9996  2868272689    91216         810            False   \n",
       "9997    9997  2868367298    91216        1245            False   \n",
       "9998    9998  2868439137    91216         711            False   \n",
       "9999    9999  2868510836    91216        1123            False   \n",
       "\n",
       "      task_container_id  answered_correctly  prior_question_elapsed_time  \n",
       "0                     1                   1                          NaN  \n",
       "1                     2                   1                      37000.0  \n",
       "2                     0                   1                      55000.0  \n",
       "3                     3                   1                      19000.0  \n",
       "4                     4                   1                      11000.0  \n",
       "...                 ...                 ...                          ...  \n",
       "9995                775                   0                      18000.0  \n",
       "9996                776                   1                      18000.0  \n",
       "9997                777                   1                      17000.0  \n",
       "9998                778                   1                      17000.0  \n",
       "9999                779                   1                      16000.0  \n",
       "\n",
       "[10000 rows x 8 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocess "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df[train_df.content_type_id == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df=train_df.sort_values(['user_id','timestamp'], ascending=True).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_df['content_type_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "      <th>content_id</th>\n",
       "      <th>task_container_id</th>\n",
       "      <th>answered_correctly</th>\n",
       "      <th>prior_question_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>115</td>\n",
       "      <td>5692</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>56943</td>\n",
       "      <td>115</td>\n",
       "      <td>5716</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>118363</td>\n",
       "      <td>115</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>55000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>131167</td>\n",
       "      <td>115</td>\n",
       "      <td>7860</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>137965</td>\n",
       "      <td>115</td>\n",
       "      <td>7922</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9758</th>\n",
       "      <td>9995</td>\n",
       "      <td>2868187305</td>\n",
       "      <td>91216</td>\n",
       "      <td>1124</td>\n",
       "      <td>775</td>\n",
       "      <td>0</td>\n",
       "      <td>18000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9759</th>\n",
       "      <td>9996</td>\n",
       "      <td>2868272689</td>\n",
       "      <td>91216</td>\n",
       "      <td>810</td>\n",
       "      <td>776</td>\n",
       "      <td>1</td>\n",
       "      <td>18000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9760</th>\n",
       "      <td>9997</td>\n",
       "      <td>2868367298</td>\n",
       "      <td>91216</td>\n",
       "      <td>1245</td>\n",
       "      <td>777</td>\n",
       "      <td>1</td>\n",
       "      <td>17000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9761</th>\n",
       "      <td>9998</td>\n",
       "      <td>2868439137</td>\n",
       "      <td>91216</td>\n",
       "      <td>711</td>\n",
       "      <td>778</td>\n",
       "      <td>1</td>\n",
       "      <td>17000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9762</th>\n",
       "      <td>9999</td>\n",
       "      <td>2868510836</td>\n",
       "      <td>91216</td>\n",
       "      <td>1123</td>\n",
       "      <td>779</td>\n",
       "      <td>1</td>\n",
       "      <td>16000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9763 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      row_id   timestamp  user_id  content_id  task_container_id  \\\n",
       "0          0           0      115        5692                  1   \n",
       "1          1       56943      115        5716                  2   \n",
       "2          2      118363      115         128                  0   \n",
       "3          3      131167      115        7860                  3   \n",
       "4          4      137965      115        7922                  4   \n",
       "...      ...         ...      ...         ...                ...   \n",
       "9758    9995  2868187305    91216        1124                775   \n",
       "9759    9996  2868272689    91216         810                776   \n",
       "9760    9997  2868367298    91216        1245                777   \n",
       "9761    9998  2868439137    91216         711                778   \n",
       "9762    9999  2868510836    91216        1123                779   \n",
       "\n",
       "      answered_correctly  prior_question_elapsed_time  \n",
       "0                      1                          NaN  \n",
       "1                      1                      37000.0  \n",
       "2                      1                      55000.0  \n",
       "3                      1                      19000.0  \n",
       "4                      1                      11000.0  \n",
       "...                  ...                          ...  \n",
       "9758                   0                      18000.0  \n",
       "9759                   1                      18000.0  \n",
       "9760                   1                      17000.0  \n",
       "9761                   1                      17000.0  \n",
       "9762                   1                      16000.0  \n",
       "\n",
       "[9763 rows x 7 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13522"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[\"content_id\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number skills 13522\n"
     ]
    }
   ],
   "source": [
    "total_ex = train_df[\"content_id\"].nunique()\n",
    "total_ex = total_ex if total_ex > train_df[\"content_id\"].max() else train_df[\"content_id\"].max()\n",
    "print(\"number skills\", total_ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number tasks 4889\n"
     ]
    }
   ],
   "source": [
    "total_task = train_df[\"task_container_id\"].nunique()\n",
    "total_task = total_task if total_task > train_df[\"task_container_id\"].max() else train_df[\"task_container_id\"].max()\n",
    "print(\"number tasks\", total_task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = pd.read_csv('./data/questions.csv')\n",
    "#questions['part']=questions['part'].astype('int8')\n",
    "train_df = train_df.merge(questions[['question_id', 'part']], left_on='content_id', right_on='question_id', how='left').drop(columns=['question_id'])\n",
    "del questions\n",
    "train_df.fillna(0,inplace = True)\n",
    "total_cat = train_df['part'].nunique()\n",
    "group = train_df[['user_id', 'content_id', 'answered_correctly','part','timestamp', 'task_container_id','prior_question_elapsed_time']].groupby('user_id').apply(lambda r: (r['content_id'].values, \n",
    "                                                                                                                                              r['part'].values, \n",
    "                                                                                                                                              r['timestamp'].values,\n",
    "                                                                                                                                              r['task_container_id'].values,\n",
    "                                                                                                                                              r['prior_question_elapsed_time'].values,\n",
    "                                                                                                                                              r['answered_correctly'].values))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id\n",
       "115      ([5692, 5716, 128, 7860, 7922, 156, 51, 50, 78...\n",
       "124      ([7900, 7876, 175, 1278, 2064, 2063, 2065, 336...\n",
       "2746     ([5273, 758, 5976, 236, 404, 382, 405, 873, 53...\n",
       "5382     ([5000, 3944, 217, 5844, 5965, 4990, 5235, 605...\n",
       "8623     ([3915, 4750, 6456, 3968, 6104, 5738, 6435, 54...\n",
       "8701     ([3901, 6671, 4963, 6143, 8279, 3964, 4002, 75...\n",
       "12741    ([5145, 9691, 9697, 5202, 4787, 5695, 7858, 56...\n",
       "13134    ([3926, 564, 3865, 4231, 3684, 3988, 3968, 521...\n",
       "24418    ([7900, 7876, 175, 1278, 2065, 2064, 2063, 336...\n",
       "24600    ([7900, 7876, 175, 1278, 2064, 2065, 2063, 336...\n",
       "32421    ([7900, 7876, 175, 1278, 2063, 2065, 2064, 336...\n",
       "40828    ([7900, 7876, 175, 1278, 2065, 2063, 2064, 336...\n",
       "44331    ([5542, 5697, 5748, 376, 5597, 6099, 4231, 689...\n",
       "45001    ([7900, 7876, 175, 1278, 2065, 2063, 2064, 336...\n",
       "46886    ([5059, 1207, 5880, 4777, 6417, 6407, 4527, 59...\n",
       "50132    ([4485, 4913, 9261, 5681, 5165, 3903, 6411, 53...\n",
       "51285    ([5692, 4512, 3566, 5811, 5132, 4466, 5183, 88...\n",
       "53842    ([7900, 7876, 175, 1278, 2065, 2063, 2064, 336...\n",
       "81002    ([3802, 993, 5516, 376, 5033, 4037, 6417, 5177...\n",
       "81429    ([7900, 7876, 175, 1278, 2065, 2063, 2064, 336...\n",
       "91216    ([7900, 7876, 175, 1278, 2063, 2064, 2065, 336...\n",
       "dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HyperParameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEBUG = True\n",
    "\n",
    "TEST_SIZE = 0.05\n",
    "\n",
    "MAX_SEQ = 97\n",
    "ACCEPTED_USER_CONTENT_SIZE = 10\n",
    "EMBED_SIZE = 64\n",
    "BATCH_SIZE = 64\n",
    "DROPOUT = 0.1\n",
    "HEADS_EN = 4\n",
    "HEADS_DE =4\n",
    "ENC_LAYERS = 3\n",
    "DEC_LAYERS = 3\n",
    "\n",
    "\n",
    "LR = 0.0005\n",
    "BETA1 = 0.9\n",
    "BETA2 = 0.999\n",
    "EPISLON = 1e-8\n",
    "warmup_steps = 4000\n",
    "PATIENCE = 10\n",
    "\n",
    "total_lag = 3001\n",
    "total_p = 301\n",
    "total_in = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerDataset(Dataset):\n",
    "    def __init__(self, group,  max_seq=MAX_SEQ):\n",
    "        super(TransformerDataset, self).__init__()\n",
    "        self.samples, self.max_seq = {}, max_seq\n",
    "        giveup_user=[]\n",
    "        \n",
    "        self.user_ids = []\n",
    "        for i, user_id in enumerate(group.index):\n",
    "            \n",
    "            print(f'Processed {i} users')\n",
    "            content_id, part, timestamp, task_container_id, prior_question_elapsed_time, answered_correctly = group[user_id]\n",
    "            # TIME STAMP\n",
    "            # Create LAG TIME\n",
    "            np_time = ((np.array(timestamp)-np.roll(np.array(timestamp), 1))).astype('int64')\n",
    "            #Control LAG TIME < 3000mins & > 0Millisecond\n",
    "            for i in range(len(np_time)):\n",
    "                if i>0:\n",
    "                    if np_time[i]==0:\n",
    "                        np_time[i]=np_time[i-1]\n",
    "                if np_time[i]<0:\n",
    "                    np_time[i]=3\n",
    "            np_time = (np_time/60000).astype('int64')\n",
    "            for i in range(len(np_time)):\n",
    "                if np_time[i]>3000:\n",
    "                    np_time[i]=3000\n",
    "                elif np_time[i]>10:\n",
    "                    np_time[i]=int(np_time[i]/10)*10\n",
    "            np_time = np_time.astype('int32')\n",
    "            # The least number of records in one sequence\n",
    "            # Create sequence\n",
    "            if len(content_id) >= ACCEPTED_USER_CONTENT_SIZE:\n",
    "                if len(content_id) > self.max_seq:\n",
    "                    total_questions = len(content_id)\n",
    "                    last_pos = total_questions // self.max_seq\n",
    "                    for seq in range(last_pos):\n",
    "                        index = f\"{user_id}_{seq}\"\n",
    "                        self.user_ids.append(index)\n",
    "                        start = seq * self.max_seq\n",
    "                        end = (seq + 1) * self.max_seq\n",
    "                        self.samples[index] = (content_id[start:end], part[start:end], np_time[start:end], task_container_id[start:end], prior_question_elapsed_time[start:end], answered_correctly[start:end])\n",
    "                    if len(content_id[end:]) >= ACCEPTED_USER_CONTENT_SIZE:\n",
    "                        index = f\"{user_id}_{last_pos + 1}\"\n",
    "                        self.user_ids.append(index)\n",
    "                        self.samples[index] = (content_id[end:], part[end:], np_time[end:], task_container_id[end:], prior_question_elapsed_time[end:], answered_correctly[end:])\n",
    "                else:\n",
    "                    index = f'{user_id}'\n",
    "                    self.user_ids.append(index)\n",
    "                    self.samples[index] = (content_id, part, np_time, task_container_id, prior_question_elapsed_time, answered_correctly)\n",
    "            else:\n",
    "                giveup_user.append(user_id)\n",
    "                \n",
    "                \n",
    "    def __len__(self):\n",
    "        return len(self.user_ids)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        user_id = self.user_ids[index]\n",
    "        content_id, part, time, task, p_time, answered_correctly = self.samples[user_id]\n",
    "        seq_len = len(content_id)\n",
    "        \n",
    "        \n",
    "                \n",
    "        \n",
    "        \n",
    "        content_id_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        part_id_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        time_fea = np.zeros(self.max_seq, dtype=int)\n",
    "        task_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        p_time_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        res_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        answered_correctly_seq = np.zeros(self.max_seq, dtype=int)\n",
    "        if seq_len >= self.max_seq:\n",
    "            content_id_seq[:] = content_id[-self.max_seq:]\n",
    "            part_id_seq[:] = part[-self.max_seq:]\n",
    "            time_fea[:] = time[-self.max_seq:]\n",
    "            task_seq[:] = task[-self.max_seq:]\n",
    "            p_time_seq[:] = p_time[-self.max_seq:]\n",
    "            res_seq[:] = answered_correctly[-self.max_seq:]\n",
    "            answered_correctly_seq[:] = answered_correctly[-self.max_seq:]\n",
    "        else:\n",
    "            content_id_seq[-seq_len:] = content_id\n",
    "            part_id_seq[-seq_len:] = part\n",
    "            time_fea[-seq_len:] = time\n",
    "            task_seq[-seq_len:] = task\n",
    "            p_time_seq[-seq_len:] = p_time\n",
    "            res_seq[-seq_len:] = answered_correctly\n",
    "            answered_correctly_seq[-seq_len:] = answered_correctly\n",
    "            \n",
    "        target_id = content_id_seq[1:]\n",
    "        part_id = part_id_seq[1:]\n",
    "        task_id = task_seq[1:]\n",
    "        lag_time_id = time_fea[1:]\n",
    "        p_time_id = p_time_seq[1:]\n",
    "        label = answered_correctly_seq[1:]\n",
    "        \n",
    "        res = res_seq[:-1]\n",
    "        #x += (answered_correctly_seq[:-1] == 1) * self.n_skill\n",
    "        \n",
    "        return target_id, part_id, task_id, lag_time_id, p_time_id, res, label\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, val = train_test_split(group, test_size = TEST_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0 users\n",
      "Processed 1 users\n",
      "Processed 2 users\n",
      "Processed 3 users\n",
      "Processed 4 users\n",
      "Processed 5 users\n",
      "Processed 6 users\n",
      "Processed 7 users\n",
      "Processed 8 users\n",
      "Processed 9 users\n",
      "Processed 10 users\n",
      "Processed 11 users\n",
      "Processed 12 users\n",
      "Processed 13 users\n",
      "Processed 14 users\n",
      "Processed 15 users\n",
      "Processed 16 users\n",
      "Processed 17 users\n",
      "Processed 18 users\n"
     ]
    }
   ],
   "source": [
    "train_dataset = TransformerDataset(train,  max_seq=MAX_SEQ)\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)\n",
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0 users\n",
      "Processed 1 users\n"
     ]
    }
   ],
   "source": [
    "val_dataset = TransformerDataset(val, max_seq=MAX_SEQ)\n",
    "val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)\n",
    "del val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 96]),\n",
       " torch.Size([64, 96]),\n",
       " torch.Size([64, 96]),\n",
       " torch.Size([64, 96]),\n",
       " torch.Size([64, 96]),\n",
       " torch.Size([64, 96]),\n",
       " torch.Size([64, 96]))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_batch = next(iter(train_dataloader))\n",
    "sample_batch[0].shape, sample_batch[1].shape, sample_batch[2].shape, sample_batch[3].shape, sample_batch[4].shape, sample_batch[5].shape, sample_batch[6].shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13522"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_ex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TransformerModel(\n",
       "  (encoder): Encoder(\n",
       "    (embedding_id): Embedding(13523, 64)\n",
       "    (pos_embedding): Embedding(96, 64)\n",
       "    (embedding_part): Embedding(8, 64)\n",
       "    (layers): ModuleList(\n",
       "      (0): TransformerBlock_en(\n",
       "        (multi_att): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_q): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_k): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_v): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (layer_normal_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (1): TransformerBlock_en(\n",
       "        (multi_att): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_q): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_k): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_v): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (layer_normal_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (2): TransformerBlock_en(\n",
       "        (multi_att): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_q): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_k): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_v): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (layer_normal_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (embedding_in): Embedding(2, 64)\n",
       "    (embedding_task): Embedding(4889, 64)\n",
       "    (embedding_lag): Embedding(3001, 64)\n",
       "    (pos_embedding): Embedding(96, 64)\n",
       "    (p_time_con): Linear(in_features=1, out_features=64, bias=False)\n",
       "    (layers): ModuleList(\n",
       "      (0): TransformerBlock_de(\n",
       "        (multi_att_1): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (multi_att_2): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout_1): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_2): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_3): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_de_in): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_en_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_de_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (1): TransformerBlock_de(\n",
       "        (multi_att_1): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (multi_att_2): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout_1): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_2): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_3): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_de_in): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_en_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_de_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (2): TransformerBlock_de(\n",
       "        (multi_att_1): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (multi_att_2): MultiheadAttention(\n",
       "          (out_proj): _LinearWithBias(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (dropout_1): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_2): Dropout(p=0.1, inplace=False)\n",
       "        (dropout_3): Dropout(p=0.1, inplace=False)\n",
       "        (layer_normal_de_in): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_en_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_de_out): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (layer_normal_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): FFN(\n",
       "          (lr1): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (relu): ReLU()\n",
       "          (bn): BatchNorm1d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (lr2): Linear(in_features=64, out_features=64, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (pred): Linear(in_features=64, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def create_model():\n",
    "    return TransformerModel(total_ex+1, total_cat, total_in, total_task, total_lag, total_p, embed_dim=EMBED_SIZE, heads_en = HEADS_EN, heads_de = HEADS_DE, max_seq=MAX_SEQ, dropout=DROPOUT, forward_expansion=1, enc_layers = ENC_LAYERS, dec_layers = DEC_LAYERS,  )\n",
    "model = create_model()\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13522"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(np.hstack(sample_batch[0].detach().numpy())).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-5.5498e-01, -2.8253e-01, -6.3585e-01,  ...,  2.8382e-01,\n",
       "         -3.0115e-01,  1.4852e-01],\n",
       "        [-3.5992e-02,  2.9492e-01, -8.6183e-01,  ..., -8.1282e-01,\n",
       "         -2.6167e-01, -4.7813e-01],\n",
       "        [ 1.8178e-01, -3.3851e-01,  2.3834e-01,  ..., -4.8998e-01,\n",
       "          2.2559e-01,  6.3762e-01],\n",
       "        ...,\n",
       "        [ 1.4564e+00,  1.0751e-01,  5.4727e-01,  ..., -9.9677e-01,\n",
       "         -7.8360e-01, -2.5469e-01],\n",
       "        [-5.0359e-01, -3.1266e-01,  4.3622e-02,  ..., -1.8906e-01,\n",
       "          1.3953e-01, -8.2845e-01],\n",
       "        [ 7.7356e-04,  7.4606e-01,  3.3155e-01,  ..., -1.6192e-01,\n",
       "          5.9918e-02, -1.2333e+00]], grad_fn=<SqueezeBackward1>)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(sample_batch[0], sample_batch[1], sample_batch[2], sample_batch[3], sample_batch[4], sample_batch[5])[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if DEBUG:\n",
    "    EPOCHS = 3\n",
    "    MODEL_PATH = './transformer_debug.pth'\n",
    "else:\n",
    "    EPOCHS = 60\n",
    "    MODEL_PATH = './transformer.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_from_item(item):\n",
    "    \n",
    "    e_id = item[0].to(device).long()\n",
    "    part_id = item[1].to(device).long()\n",
    "    task_id = item[2].to(device).long()\n",
    "    time_id = item[3].to(device).long()\n",
    "    p_time = item[4].to(device).long()\n",
    "    res = item[5].to(device).long()\n",
    "    label = item[6].to(device).float()\n",
    "    target_mask = (part_id != 0)\n",
    "    return e_id, part_id, task_id, time_id, p_time, res, label, target_mask\n",
    "\n",
    "def update_stats(tbar, train_loss, loss, output, label, num_corrects, num_total, labels, outs):\n",
    "    train_loss.append(loss.item())\n",
    "    pred = (torch.sigmoid(output) >= 0.5).long()\n",
    "    num_corrects += (pred == label).sum().item()\n",
    "    num_total += len(label)\n",
    "    labels.extend(label.view(-1).data.cpu().numpy())\n",
    "    outs.extend(output.view(-1).data.cpu().numpy())\n",
    "    tbar.set_description('loss - {:.4f}'.format(loss))\n",
    "    return num_corrects, num_total\n",
    "\n",
    "def train_epoch(model, dataloader, optim, criterion,device=\"cpu\"):\n",
    "    model.train()\n",
    "    \n",
    "    train_loss = []\n",
    "    num_corrects = 0\n",
    "    num_total = 0\n",
    "    labels = []\n",
    "    outs = []\n",
    "    \n",
    "    tbar = tqdm(dataloader)\n",
    "    for item in tbar:\n",
    "        e_id, part_id, task_id, lag_time, p_time, res, label, target_mask = load_from_item(item)\n",
    "        \n",
    "        optim.optimizer.zero_grad()\n",
    "        output, _ = model(e_id, part_id, task_id, lag_time, p_time, res)\n",
    "        \n",
    "        output = torch.masked_select(output, target_mask)\n",
    "        label = torch.masked_select(label, target_mask)\n",
    "        \n",
    "        loss = criterion(output, label)\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        #scheduler.step()\n",
    "        \n",
    "        tbar.set_description('loss - {:.4f}'.format(loss))\n",
    "        #auc = roc_auc_score(labels, outs)\n",
    "\n",
    "def val_epoch(model, val_iterator, criterion, device=\"cpu\"):\n",
    "    model.eval()\n",
    "\n",
    "    train_loss = []\n",
    "    num_corrects = 0\n",
    "    num_total = 0\n",
    "    labels = []\n",
    "    outs = []\n",
    "    \n",
    "    tbar = tqdm(val_iterator)\n",
    "    for item in tbar:\n",
    "        e_id, part_id, task_id, lag_time, p_time, res, label, target_mask = load_from_item(item)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            output, atten_weight = model(e_id, part_id, task_id, lag_time, p_time, res)\n",
    "        \n",
    "        output = torch.masked_select(output, target_mask)\n",
    "        label = torch.masked_select(label, target_mask)\n",
    "\n",
    "        loss = criterion(output, label)\n",
    "        \n",
    "        num_corrects, num_total = update_stats(tbar, train_loss, loss, output, label, num_corrects, num_total, labels, outs)\n",
    "\n",
    "    acc = num_corrects / num_total\n",
    "    auc = roc_auc_score(labels, outs)\n",
    "    loss = np.average(train_loss)\n",
    "\n",
    "    return loss, acc, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_std_opt(d_model, warmup_step, LR, B1, B2, EPS):\n",
    "    return NoamOpt(d_model, warmup_step,\n",
    "            torch.optim.Adam(model.parameters(), lr=LR, betas=(B1, B2), eps=EPS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_train():\n",
    "    optimizer = get_std_opt(EMBED_SIZE, warmup_steps, LR, BETA1, BETA2, EPISLON)\n",
    "    #optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR, steps_per_epoch=len(train_dataloader), epochs=EPOCHS)\n",
    "    earlystopping = EarlyStopping(PATIENCE, verbose=True)\n",
    "    \n",
    "    model.to(device)\n",
    "    criterion.to(device)\n",
    "    best_auc = 0.0\n",
    "    for epoch in range(EPOCHS):\n",
    "        train_epoch(model, train_dataloader, optimizer, criterion, device)\n",
    "        val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)\n",
    "        print(f\"epoch - {epoch + 1} val_loss - {val_loss:.3f} acc - {avl_acc:.3f} auc - {val_auc:.3f}\")\n",
    "        if best_auc < val_auc:\n",
    "            print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')\n",
    "            best_auc = val_auc\n",
    "            torch.save(model.state_dict(), MODEL_PATH)\n",
    "        print('LR is:', optimizer.optimizer.param_groups[0]['lr'])\n",
    "        earlystopping(val_loss, model)\n",
    "        if earlystopping.early_stop:\n",
    "            print('early stopping')\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9646ee99c8ad4135b20f2840aa38d1b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dca8e002e6c84bff9198eb4ceb885fa7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch - 1 val_loss - 0.721 acc - 0.463 auc - 0.517\n",
      "epoch - 1 best model with val auc: 0.5169082125603865\n",
      "LR is: 9.882117688026186e-07\n",
      "Validation loss decreased (inf --> 0.720676).  Saving model ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96d9b3f69d774803ad4726bbd87c2794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b22e60ef13b74fd0af1b73b71d0d7a32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch - 2 val_loss - 0.722 acc - 0.463 auc - 0.534\n",
      "epoch - 2 best model with val auc: 0.5338164251207729\n",
      "LR is: 1.976423537605237e-06\n",
      "EarlyStopping counter: 1 out of 10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7812036aff2b42999de1a99218f381e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66c741267f7e454db3200eceb4224a4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch - 3 val_loss - 0.723 acc - 0.463 auc - 0.536\n",
      "epoch - 3 best model with val auc: 0.5362318840579711\n",
      "LR is: 2.964635306407856e-06\n",
      "EarlyStopping counter: 2 out of 10\n"
     ]
    }
   ],
   "source": [
    "do_train()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}