workspace_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RECORDING_SESSION_LABEL', 'trial', 'IA_ID', 'item', 'list', 'IA_LABEL',\n",
       "       'wordlength', 'condition', 'is_critical', 'is_spill1', 'is_spill2',\n",
       "       'is_spill3', 'filler', 'LF', 'HF', 'function_word', 'other_filler',\n",
       "       'composite', 'fixation_duration', 'duration_firstpass',\n",
       "       'duration_firstfixation', 'fix_count', 'avg_pupil',\n",
       "       'IA_REGRESSION_IN_COUNT', 'IA_REGRESSION_OUT_COUNT', 'saccade_length',\n",
       "       'saccade_duration', 'go_past_time', 'sentenceCondition'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from itertools import product\n",
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix\n",
    "import torch.nn.utils.rnn as RNN\n",
    "import torch.nn.functional as F\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "seed = 42\n",
    "\n",
    "data = pd.read_csv('data.csv', delimiter=';')\n",
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "343\n"
     ]
    }
   ],
   "source": [
    "filtered = data.copy()\n",
    "\n",
    "filtered[\"sentenceCondition\"] = filtered[\"sentenceCondition\"].map(lambda x: x.replace(\"none\", \"2\"))\n",
    "filtered[\"sentenceCondition\"] = filtered[\"sentenceCondition\"].map(lambda x: x.replace(\"control\", \"0\"))\n",
    "filtered[\"sentenceCondition\"] = filtered[\"sentenceCondition\"].map(lambda x: x.replace(\"pseudo\", \"1\"))\n",
    "filtered[\"sentenceCondition\"] = filtered[\"sentenceCondition\"].map(lambda x: x.replace(\"filler\", \"3\"))\n",
    "\n",
    "filtered['attention'] = filtered['condition'].copy()\n",
    "\n",
    "filtered['condition'] = filtered['condition'].map(lambda x: x.replace(\"control\", \"0\"))\n",
    "filtered['condition'] = filtered['condition'].map(lambda x: x.replace(\"pseudo\", \"1\"))\n",
    "filtered['condition'] = filtered['condition'].map(lambda x: x.replace(\"filler\", \"2\"))\n",
    "filtered['condition'] = filtered['condition'].map(lambda x: x.replace(\"none\", \"3\"))\n",
    "\n",
    "filtered['attention'] = filtered['attention'].map(lambda x: x.replace(\"control\", \"1\"))\n",
    "filtered['attention'] = filtered['attention'].map(lambda x: x.replace(\"pseudo\", \"1\"))\n",
    "filtered['attention'] = filtered['attention'].map(lambda x: x.replace(\"filler\", \"0\"))\n",
    "filtered['attention'] = filtered['attention'].map(lambda x: x.replace(\"none\", \"0\"))\n",
    "\n",
    "\n",
    "filtered[\"sentenceCondition\"] = filtered[\"sentenceCondition\"].astype(int)\n",
    "filtered['condition'] = filtered['condition'].astype(int)\n",
    "filtered['attention'] = filtered['attention'].astype(int)\n",
    "\n",
    "control = filtered.loc[filtered['sentenceCondition'] == 0].copy()\n",
    "pseudo = filtered.loc[filtered['sentenceCondition'] == 1].copy()\n",
    "mapped = pd.concat([control, pseudo])\n",
    "\n",
    "\n",
    "mapped.drop([\"IA_ID\", \"item\", \"list\", \"IA_LABEL\"], axis=1, inplace=True)\n",
    "normalized = mapped[['fixation_duration',\n",
    "       'duration_firstpass', 'duration_firstfixation', 'fix_count',\n",
    "       'avg_pupil', 'IA_REGRESSION_IN_COUNT', 'IA_REGRESSION_OUT_COUNT',\n",
    "       'saccade_length', 'saccade_duration', 'go_past_time']]\n",
    "normalized = (normalized - normalized.mean()) / normalized.std()\n",
    "mapped[['fixation_duration',\n",
    "       'duration_firstpass', 'duration_firstfixation', 'fix_count',\n",
    "       'avg_pupil', 'IA_REGRESSION_IN_COUNT', 'IA_REGRESSION_OUT_COUNT',\n",
    "       'saccade_length', 'saccade_duration', 'go_past_time']] = normalized\n",
    "sentences = mapped.groupby(['RECORDING_SESSION_LABEL', 'trial'])\n",
    "print(len(sentences))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(343, 18) (343, 18, 22) (343, 18) (343,)\n"
     ]
    }
   ],
   "source": [
    "label_array = list()\n",
    "features_array = list()\n",
    "attention_mask_array = list()\n",
    "for item in sentences:\n",
    "    label_array.append(item[1][\"condition\"])\n",
    "    attention_mask_array.append(item[1]['attention'])\n",
    "    features = item[1].drop(['RECORDING_SESSION_LABEL', 'trial', 'sentenceCondition', 'condition'], axis=1).to_numpy()\n",
    "    features = (features - features.mean()) / features.std()\n",
    "    #print(features.shape)\n",
    "    features_array.append(features)\n",
    "\n",
    "def pad_matrix_to_same_size(lists):\n",
    "    maxlen = max([len(l) for l in lists])\n",
    "    return [np.concatenate((np.zeros((maxlen - l.shape[0], l.shape[1])), l), axis=0) for l in lists]\n",
    "\n",
    "def pad_series_to_same_size(lists):\n",
    "    maxlen = max([len(l) for l in lists])\n",
    "    return [np.concatenate((np.zeros((maxlen - len(l))), l), axis=0) for l in lists]\n",
    "\n",
    "lengths = np.array([len(l) for l in features_array])\n",
    "padded_features_array = np.array(pad_matrix_to_same_size(features_array))\n",
    "padded_attention_mask_array = np.array(pad_series_to_same_size(attention_mask_array))\n",
    "padded_label_array = np.array(pad_series_to_same_size(label_array))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, features, labels, attention_mask):\n",
    "        self.features = features\n",
    "        self.labels = labels\n",
    "        self.attention_mask = attention_mask\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        features = self.features[index]\n",
    "        label = self.labels[index]\n",
    "        attention_mask = self.attention_mask[index]\n",
    "        return features, label, attention_mask\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.features)\n",
    "\n",
    "dataset = CustomDataset(features=padded_features_array, labels=padded_label_array, attention_mask=padded_attention_mask_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_fold_split_data(dataset, batch_size, k=5):\n",
    "    n = len(dataset)\n",
    "    fold_size = n // k\n",
    "    folds = []\n",
    "    for i in range(k):\n",
    "        start = i * fold_size\n",
    "        end = (i + 1) * fold_size if i < k - 1 else n\n",
    "        folds.append(torch.utils.data.Subset(dataset, range(start, end)))\n",
    "\n",
    "    dataloaders = []\n",
    "    for i in range(k):\n",
    "        validation_dataset = folds[i]\n",
    "        train_folds = [folds[j] for j in range(k) if j != i]\n",
    "        train_dataset = torch.utils.data.ConcatDataset(train_folds)\n",
    "\n",
    "        train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "        validation_dataloader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=True)\n",
    "        dataloaders.append((train_dataloader, validation_dataloader))\n",
    "\n",
    "    return dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test(model, dataloader, optimizer, training=\"train\"):\n",
    "   \n",
    "    loss_function = torch.nn.BCEWithLogitsLoss()\n",
    "\n",
    "    if training == \"train\":\n",
    "        model.train()\n",
    "    elif training == \"validation\":\n",
    "        model.eval()\n",
    "    elif training == \"test\":\n",
    "        model.eval()\n",
    "    else:\n",
    "        raise ValueError(\"training argument must be either 'train', 'validation' or 'test'\")\n",
    "        \n",
    "    cumulative_loss = 0\n",
    "    prediction_list = []\n",
    "    label_list = []\n",
    "    sigmoid = torch.nn.Sigmoid()\n",
    "    \n",
    "    for sample in dataloader:\n",
    "        input, targets, attention_mask = sample[0].float().to(device), sample[1].type(torch.LongTensor).to(device) , sample[2].to(device)\n",
    "        output = model(input, attention_mask).to(device)\n",
    "        predictions = output#[attention_mask == 1]\n",
    "\n",
    "        targets = targets[attention_mask == 1]\n",
    "        loss_value = loss_function(predictions, targets.unsqueeze(1).float())\n",
    "        cumulative_loss += loss_value.item()\n",
    "\n",
    "        if training == \"train\":\n",
    "            optimizer.zero_grad()\n",
    "            loss_value.sum().backward()\n",
    "            optimizer.step()\n",
    "        predictions = [round(x) for x in sigmoid(predictions.squeeze(1)).to('cpu').detach().numpy().tolist()]\n",
    "        \n",
    "        target_labels = sample[1][attention_mask.to('cpu') == 1]\n",
    "        prediction_list.extend(predictions)\n",
    "        label_list.extend(target_labels)\n",
    "\n",
    "    if training == \"test\":\n",
    "        print(confusion_matrix(label_list, prediction_list))\n",
    "        return label_list, prediction_list\n",
    "    f1 = f1_score(label_list, prediction_list)\n",
    "    accuracy = accuracy_score(label_list, prediction_list)\n",
    "    confusion = confusion_matrix(label_list, prediction_list)\n",
    "\n",
    "    return cumulative_loss, accuracy, f1, confusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TuneableModel(torch.nn.Module):\n",
    "    def __init__(self, input_size, layer_size, dropout_rate, n_layers):\n",
    "        super(TuneableModel, self).__init__()\n",
    "        self.lstm = torch.nn.LSTM(input_size=input_size, hidden_size=layer_size, bidirectional=False, \n",
    "                                  num_layers=n_layers, batch_first=True, dropout=dropout_rate)#, proj_size=1)\n",
    "        self.output_layer = torch.nn.Linear(layer_size, 1)\n",
    "        self.batchnorm = torch.nn.BatchNorm1d(layer_size)\n",
    "        self.activation = torch.nn.LeakyReLU()\n",
    "        self.linear = torch.nn.Linear(layer_size, layer_size)\n",
    "\n",
    "    def forward(self, x, attention_mask):\n",
    "        x = self.lstm(x)\n",
    "        x = self.activation(x[0][:, -1, :])\n",
    "        x = self.output_layer(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training sample\n",
    "def evaluate(params):\n",
    "    dropout, hidden_size, learning_rate, batch_size, n_hidden = params\n",
    "\n",
    "    max_epochs = 1000\n",
    "    max_patience = 10\n",
    "    \n",
    "    predictions = []\n",
    "    labels = []\n",
    "    \n",
    "    dataloaders = k_fold_split_data(dataset, batch_size, k=10)\n",
    "    for i, dataloader in tqdm(enumerate(dataloaders)):\n",
    "        train_dataloader, validation_dataloader = dataloader[0], dataloader[1]\n",
    "        test_dataloader = dataloader[1]\n",
    "        PATH = f\"model_{i}.pt\"\n",
    "        last_loss = 1000000\n",
    "        torch.manual_seed(seed)\n",
    "        input_size = train_dataloader.dataset[0][0].shape[1]\n",
    "        model = TuneableModel(input_size, hidden_size, dropout, n_hidden)\n",
    "        model.to(device)\n",
    "        optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, betas=(0.99, 0.99), weight_decay=1e-2)\n",
    "\n",
    "        for epoch in range(max_epochs):\n",
    "            # training\n",
    "            train_loss, train_accuracy, train_f1, train_confusion = train_test(model, train_dataloader, optimizer, training=\"train\")\n",
    "            train_loss, train_accuracy, train_f1 = train_loss, round(train_accuracy, 4), round(train_f1, 2)\n",
    "            # validation at end of epoch\n",
    "            validation_loss, validation_accuracy, validation_f1, validation_confusion = train_test(model, validation_dataloader, optimizer, training=\"validation\")\n",
    "            validation_loss, validation_accuracy, validation_f1 = validation_loss, round(validation_accuracy, 4), round(validation_f1, 2)\n",
    "            if validation_loss < last_loss:\n",
    "                last_loss = validation_loss\n",
    "                current_patience = 0\n",
    "            else:\n",
    "                if current_patience == 0:\n",
    "                    torch.save({\n",
    "                        'epoch': epoch,\n",
    "                        'model_state_dict': model.state_dict(),\n",
    "                        'optimizer_state_dict': optimizer.state_dict(),\n",
    "                        'loss': last_loss,\n",
    "                        }, PATH)\n",
    "                current_patience += 1\n",
    "            if current_patience == max_patience:\n",
    "                break   \n",
    "\n",
    "        # Testing once patience is reached\n",
    "        torch.manual_seed(seed)\n",
    "        model = TuneableModel(input_size, hidden_size, dropout, n_hidden)\n",
    "        model.to(device)\n",
    "        optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, betas=(0.99, 0.99), weight_decay=1e-2)\n",
    "        checkpoint = torch.load(PATH)\n",
    "        model.load_state_dict(checkpoint['model_state_dict'])\n",
    "        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "        prediction_list, label_list = train_test(model, test_dataloader, optimizer, training=\"test\")\n",
    "        predictions.extend(prediction_list)\n",
    "        labels.extend(label_list)\n",
    "        \n",
    "    return accuracy_score(labels, predictions), f1_score(labels, predictions), confusion_matrix(labels, predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:23, 23.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[26  0]\n",
      " [ 2  6]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:40, 19.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[23  3]\n",
      " [ 2  6]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:55, 17.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[25  0]\n",
      " [ 3  6]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4it [01:12, 17.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[24  1]\n",
      " [ 0  9]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [01:25, 15.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[24  1]\n",
      " [ 4  5]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [01:41, 15.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[21  4]\n",
      " [ 0  9]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7it [02:02, 17.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[24  1]\n",
      " [ 1  8]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [02:13, 15.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[24  2]\n",
      " [ 5  3]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9it [02:28, 15.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[28  0]\n",
      " [ 1  5]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [02:54, 17.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[25  2]\n",
      " [ 2  8]]\n",
      "acc: 90.09%\n",
      " f1: 0.793\n",
      "[[244  20]\n",
      " [ 14  65]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "params = (0.0, 10, 0.01, 32, 1) \n",
    "accuracy, f1, confusion = evaluate(params)\n",
    "print(f\"acc: {round(accuracy*100,2)}%\\n f1: {round(f1,3)}\")\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1024\n"
     ]
    }
   ],
   "source": [
    "params_nn ={\n",
    "    'dropout': [x/10 for x in list(range(0, 10, 3))],\n",
    "    'hidden_size': list(range(0, 101, 25))[1:],\n",
    "    'learning_rate': [0.01, 0.001, 0.0001, 1e-05],\n",
    "    'batch_size': [2*2**x for x in range(2, 6)],\n",
    "    'n_hidden': list(range(1, 5, 1))\n",
    "}\n",
    "parameter_expansion = list(product(*params_nn.values()))\n",
    "print(len(parameter_expansion))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[26  0]\n",
      " [ 2  6]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:18, 18.93s/it]\n",
      "0it [00:18, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[28], line 4\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, p \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28menumerate\u001b[39m(parameter_expansion)):\n\u001b[0;32m      3\u001b[0m     dropout, hidden_size, learning_rate, batch_size, n_hidden \u001b[38;5;241m=\u001b[39m p\n\u001b[1;32m----> 4\u001b[0m     accuracy, f1, confusion \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      5\u001b[0m     model_performance \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdropout\u001b[39m\u001b[38;5;124m\"\u001b[39m: dropout, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhidden_size\u001b[39m\u001b[38;5;124m\"\u001b[39m: hidden_size, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlearning_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m: learning_rate, \n\u001b[0;32m      6\u001b[0m               \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbatch_size\u001b[39m\u001b[38;5;124m\"\u001b[39m: batch_size, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mn_hidden\u001b[39m\u001b[38;5;124m\"\u001b[39m: n_hidden, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maccuracy\u001b[39m\u001b[38;5;124m\"\u001b[39m: accuracy, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mf1\u001b[39m\u001b[38;5;124m\"\u001b[39m: f1}\n\u001b[0;32m      7\u001b[0m     results[i] \u001b[38;5;241m=\u001b[39m model_performance\n",
      "Cell \u001b[1;32mIn[25], line 26\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(params)\u001b[0m\n\u001b[0;32m     22\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdamW(model\u001b[38;5;241m.\u001b[39mparameters(), lr\u001b[38;5;241m=\u001b[39mlearning_rate)\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(max_epochs):\n\u001b[0;32m     25\u001b[0m     \u001b[38;5;66;03m# training\u001b[39;00m\n\u001b[1;32m---> 26\u001b[0m     train_loss, train_accuracy, train_f1, train_confusion \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_test\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     27\u001b[0m     train_loss, train_accuracy, train_f1 \u001b[38;5;241m=\u001b[39m train_loss, \u001b[38;5;28mround\u001b[39m(train_accuracy, \u001b[38;5;241m4\u001b[39m), \u001b[38;5;28mround\u001b[39m(train_f1, \u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m     28\u001b[0m     \u001b[38;5;66;03m# validation at end of epoch\u001b[39;00m\n",
      "Cell \u001b[1;32mIn[23], line 23\u001b[0m, in \u001b[0;36mtrain_test\u001b[1;34m(model, dataloader, optimizer, training)\u001b[0m\n\u001b[0;32m     21\u001b[0m targets \u001b[38;5;241m=\u001b[39m targets[attention_mask \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m]\n\u001b[0;32m     22\u001b[0m loss_value \u001b[38;5;241m=\u001b[39m loss_function(predictions, targets)\n\u001b[1;32m---> 23\u001b[0m cumulative_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43mloss_value\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m training \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m     26\u001b[0m     optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for i, p in tqdm(enumerate(parameter_expansion)):\n",
    "    dropout, hidden_size, learning_rate, batch_size, n_hidden = p\n",
    "    accuracy, f1, confusion = evaluate(p)\n",
    "    model_performance = {\"dropout\": dropout, \"hidden_size\": hidden_size, \"learning_rate\": learning_rate, \n",
    "              \"batch_size\": batch_size, \"n_hidden\": n_hidden, \"accuracy\": accuracy, \"f1\": f1}\n",
    "    results[i] = model_performance\n",
    "    print(\"Confusion matrix:\\n\", confusion)\n",
    "    print(model_performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_dataframe = pd.DataFrame.from_dict(results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}