kd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import torch\n",
    "import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.\n",
    "import torchvision.transforms as transforms  # Transformations we can perform on our dataset\n",
    "import os\n",
    "import cv2\n",
    "import pandas as pd\n",
    "from skimage import io\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "\n",
    "# import scripts\n",
    "from scripts.saveResults import  *\n",
    "from torch.utils.data import (\n",
    "    Dataset,\n",
    "    DataLoader,\n",
    ")  # Gives easier dataset managment and creates mini batches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class SolarRadiance(Dataset):\n",
    "    def __init__(self, root_dir, labels, transform):\n",
    "        self.root_dir = root_dir\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        # self.data = self.load_dataset()\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        img_path = self.labels.iloc[index,0]\n",
    "        target = self.labels.iloc[index,1]\n",
    "\n",
    "        image = io.imread(img_path)\n",
    "        image = cv2.imread(img_path)#, cv2.IMREAD_COLOR)\n",
    "        image = cv2.resize(image, (60, 80))\n",
    "\n",
    "        if self.transform:\n",
    "             image = self.transform(image)\n",
    "\n",
    "        y_label = torch.tensor(target)\n",
    "\n",
    "        return image, y_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(root_dir):\n",
    "        ds = pd.DataFrame()\n",
    "        dates = os.listdir(root_dir)\n",
    "\n",
    "        try:\n",
    "            for date in dates:\n",
    "                infrared_folder = os.path.join(root_dir, date, \"infrared\")\n",
    "                pyranometer_folder = os.path.join(root_dir, date, \"pyranometer\")\n",
    "                csv_path = os.path.join(pyranometer_folder, \"{date}.csv\".format(date=date))\n",
    "                if not os.path.exists(csv_path):\n",
    "                    print(\"Skipping date {date} because it does not have both infrared and pyranometer folders\".format(date=date))\n",
    "                    continue\n",
    "                ds_temp = getDs(infrared_folder, csv_path)\n",
    "                \n",
    "                # append the dataframe in the final dataframe\n",
    "                # ds = ds.append(ds_temp)\n",
    "                ds = pd.concat([ds, ds_temp], ignore_index=True)\n",
    "\n",
    "                ds['name'] = ds['name'].apply(lambda img: os.path.join(root_dir, date, 'infrared', img))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "        return ds\n",
    "\n",
    "def getDs(path, labels):\n",
    "    pyranometer = pd.read_csv(labels)\n",
    "    images = os.listdir(path)\n",
    "    \n",
    "   \n",
    "    #convert column 1 to int\n",
    "    X = pyranometer.iloc[:,0].astype(int)\n",
    "\n",
    "    #convert to image names\n",
    "    pyranometer.iloc[:,0] = X.apply(lambda x: str(x) + 'IR.png')\n",
    "    \n",
    "    # Filter pyranometer DataFrame based on the 'x' column\n",
    "\n",
    "    filtered_pyranometer = pyranometer[pyranometer.iloc[:,0].isin(images)]\n",
    "    # Display the result\n",
    "    filtered_pyranometer.columns = ['name', 'value']\n",
    "\n",
    "    filtered_pyranometer = filtered_pyranometer.drop_duplicates(subset='name')\n",
    "    return filtered_pyranometer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set device\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Hyperparameters\n",
    "in_channel = 1\n",
    "batch_size = 50\n",
    "num_epochs = 50\n",
    "loss = 1 # if loss = 0 the model will be trained with RMSE loss and vice versa\n",
    "lr=0.01 # learning rate\n",
    "random_seed = 42\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create result directory\n",
    "result_dir = 'result/kd'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = 'F:/Girasol/extracted/train/'\n",
    "train_data  = load_dataset(train_dir)\n",
    "torch.manual_seed(random_seed)\n",
    "train_set = SolarRadiance(root_dir= train_dir, labels=train_data, transform = transforms.Compose([transforms.ToTensor()]))\n",
    "train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "val_dir = 'F:/Girasol/extracted/val/'\n",
    "val_data  = load_dataset(val_dir)\n",
    "torch.manual_seed(random_seed)\n",
    "random_seed = 42\n",
    "val_set = SolarRadiance(root_dir= val_dir, labels=val_data, transform = transforms.Compose([transforms.ToTensor()]))\n",
    "val_loader = DataLoader(dataset=val_set, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "class CNNRegression(nn.Module):\n",
    "    def __init__(self, image_size, num_channels):\n",
    "        super(CNNRegression, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(num_channels, 16, kernel_size=3, stride=1, padding=1)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        self.fc1 = nn.Linear(int(60/4) * int(80/4) * 32, 64)\n",
    "        self.relu3 = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(64, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.relu1(x)\n",
    "        x = self.pool1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = self.relu2(x)\n",
    "        x = self.pool2(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc1(x)\n",
    "        x = self.relu3(x)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNNRegression(\n",
       "  (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (relu1): ReLU()\n",
       "  (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (conv2): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (relu2): ReLU()\n",
       "  (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (fc1): Linear(in_features=9600, out_features=64, bias=True)\n",
       "  (relu3): ReLU()\n",
       "  (fc2): Linear(in_features=64, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "student = CNNRegression(60, 3).to(device)\n",
    "\n",
    "# Load the pre-trained model\n",
    "# student.load_state_dict(torch.load('results/batch_50_lr_0.01_loss_1/2024-01-01_00-29-03/best_model.pth'))\n",
    "student.to(device)\n",
    "student.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loss and Optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if loss == 0:\n",
    "    # create a function (this my favorite choice)\n",
    "    def RMSELoss(yhat,y):\n",
    "        return torch.sqrt(torch.mean((yhat-y)**2))\n",
    "    criterion = RMSELoss\n",
    "else:\n",
    "    # Define the model, loss function, and optimizer\n",
    "    criterion = nn.MSELoss()\n",
    "    \n",
    "\n",
    "optimizer = torch.optim.Adam(student.parameters(), lr=lr)\n",
    "# Create a StepLR scheduler\n",
    "scheduler = StepLR(optimizer, step_size=1, gamma=0.5)\n",
    "\n",
    "# set initial lowest_loss to an infinite number\n",
    "lowest_loss = 10000000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\nijhu\\anaconda3\\envs\\pytorch\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "c:\\Users\\nijhu\\anaconda3\\envs\\pytorch\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=MobileNet_V2_Weights.IMAGENET1K_V1`. You can also use `weights=MobileNet_V2_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.models as models\n",
    "\n",
    "# Load the pre-trained MobileNet model without its final classification layer\n",
    "mobilenet = models.mobilenet_v2(pretrained=True)\n",
    "# Remove the last layer (classification layer)\n",
    "mobilenet = nn.Sequential(*list(mobilenet.children())[:-1])\n",
    "\n",
    "# Add custom layers for regression\n",
    "class LinearRegressionHead(nn.Module):\n",
    "    def __init__(self, in_features):\n",
    "        super(LinearRegressionHead, self).__init__()\n",
    "        self.fc = nn.Linear(in_features, 1)  # Output 1 value for linear regression\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "# Append the custom regression head to the pre-trained model\n",
    "in_features = mobilenet[-1][-1].out_channels  # Get the number of output channels from the last layer of MobileNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\nijhu\\anaconda3\\envs\\pytorch\\Lib\\site-packages\\torch\\optim\\lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [05:48<00:00,  2.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2362.7209, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1977.1278, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2285.6023, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 5125.0253\n",
      "Mean_loss 5125.025346427128lowest_loss10000000\n",
      "RMSE LOSS: tensor(71.5893)\n",
      "Epoch  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:49<00:00,  8.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1904.4681, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1467.2662, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1817.0278, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 3783.5929\n",
      "Mean_loss 3783.5929223422345lowest_loss5125.025346427128\n",
      "RMSE LOSS: tensor(61.5109)\n",
      "Epoch  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:51<00:00,  8.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3000.9700, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(3773.3176, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3155.4397, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 3180.6226\n",
      "Mean_loss 3180.622602791622lowest_loss3783.5929223422345\n",
      "RMSE LOSS: tensor(56.3970)\n",
      "Epoch  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:51<00:00,  8.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1351.5884, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1058.2418, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1292.9191, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2934.1553\n",
      "Mean_loss 2934.1552860654633lowest_loss3180.622602791622\n",
      "RMSE LOSS: tensor(54.1678)\n",
      "Epoch  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:52<00:00,  8.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2842.7712, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1642.4373, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2602.7046, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2682.9502\n",
      "Mean_loss 2682.950215306775lowest_loss2934.1552860654633\n",
      "RMSE LOSS: tensor(51.7972)\n",
      "Epoch  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:55<00:00,  8.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1879.1448, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2868.8674, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2077.0894, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2804.6780\n",
      "Epoch  6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  8.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2035.7711, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1686.5568, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1965.9283, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2570.1013\n",
      "Mean_loss 2570.101334144329lowest_loss2682.950215306775\n",
      "RMSE LOSS: tensor(50.6962)\n",
      "Epoch  7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3253.3086, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(3059.7634, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3214.5996, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2580.7895\n",
      "Epoch  8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:54<00:00,  8.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1462.9543, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1365.2344, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1443.4104, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2553.5298\n",
      "Mean_loss 2553.529848822232lowest_loss2570.101334144329\n",
      "RMSE LOSS: tensor(50.5325)\n",
      "Epoch  9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:45<00:00,  5.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1116.2198, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1541.0087, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1201.1776, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.9078\n",
      "Mean_loss 2544.90777219575lowest_loss2553.529848822232\n",
      "RMSE LOSS: tensor(50.4471)\n",
      "Epoch  10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [03:58<00:00,  3.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(4600.7856, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(4734.2627, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(4627.4814, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.0413\n",
      "Mean_loss 2542.0412502946524lowest_loss2544.90777219575\n",
      "RMSE LOSS: tensor(50.4187)\n",
      "Epoch  11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:53<00:00,  8.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1036.6294, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1096.8400, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1048.6715, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2551.6840\n",
      "Epoch  12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1437.5032, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1514.0312, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1452.8088, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.2079\n",
      "Epoch  13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:55<00:00,  8.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2204.9543, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2225.9580, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2209.1550, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2548.5630\n",
      "Epoch  14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1293.2604, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1135.0657, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1261.6215, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2540.9584\n",
      "Mean_loss 2540.95843979408lowest_loss2542.0412502946524\n",
      "RMSE LOSS: tensor(50.4079)\n",
      "Epoch  15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2066.1396, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1227.8938, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1898.4905, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.0871\n",
      "Epoch  16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2170.0837, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2647.0081, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2265.4688, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.9826\n",
      "Epoch  17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:58<00:00,  7.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3272.1050, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(3288.7546, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3275.4351, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2540.6783\n",
      "Mean_loss 2540.678268958782lowest_loss2540.95843979408\n",
      "RMSE LOSS: tensor(50.4051)\n",
      "Epoch  18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3892.1956, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2675.0688, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3648.7705, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2546.9195\n",
      "Epoch  19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:58<00:00,  7.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1753.6007, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1606.0201, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1724.0847, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2545.6182\n",
      "Epoch  20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2026.8442, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1889.8391, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1999.4434, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2540.4398\n",
      "Mean_loss 2540.4398382778827lowest_loss2540.678268958782\n",
      "RMSE LOSS: tensor(50.4028)\n",
      "Epoch  21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:58<00:00,  7.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1338.1604, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1810.8420, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1432.6968, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2546.1073\n",
      "Epoch  22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1350.2965, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1423.2234, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1364.8818, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.7019\n",
      "Epoch  23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:17<00:00,  6.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3988.3142, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(3282.8530, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3847.2219, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2548.6041\n",
      "Epoch  24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:41<00:00,  5.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1421.5524, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2047.1008, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1546.6621, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.0298\n",
      "Epoch  25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:40<00:00,  5.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(753.2554, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1038.1532, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(810.2350, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2543.2064\n",
      "Epoch  26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:41<00:00,  5.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2195.4197, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1780.5391, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2112.4436, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2541.7155\n",
      "Epoch  27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:41<00:00,  5.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3134.3672, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2539.7908, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3015.4521, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.9216\n",
      "Epoch  28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:42<00:00,  5.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1119.2954, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1183.5608, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1132.1484, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2543.3659\n",
      "Epoch  29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:41<00:00,  5.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2872.5442, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2685.9028, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2835.2161, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.6612\n",
      "Epoch  30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:58<00:00,  5.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1696.1678, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1900.2139, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1736.9771, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.6027\n",
      "Epoch  31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:46<00:00,  5.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3848.6575, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2559.4255, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(3590.8110, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.0797\n",
      "Epoch  32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:37<00:00,  5.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1929.4810, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1536.2279, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1850.8304, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2546.0348\n",
      "Epoch  33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:39<00:00,  5.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2497.8425, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2905.3259, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2579.3394, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.4788\n",
      "Epoch  34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:08<00:00,  7.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1756.5646, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1315.2231, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1668.2964, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.1497\n",
      "Epoch  35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2612.2158, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2641.3340, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2618.0396, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.3265\n",
      "Epoch  36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1554.5985, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1346.6781, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1513.0144, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2540.4822\n",
      "Epoch  37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1695.5679, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1617.9366, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1680.0417, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.0168\n",
      "Epoch  38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:57<00:00,  7.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1583.8427, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1418.6838, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1550.8109, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.0320\n",
      "Epoch  39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  8.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(996.8000, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1247.7328, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1046.9866, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2540.4651\n",
      "Epoch  40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:55<00:00,  8.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2428.7825, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2415.3484, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2426.0957, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2543.4714\n",
      "Epoch  41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:54<00:00,  8.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2222.6292, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1991.0747, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2176.3184, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.1931\n",
      "Epoch  42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:58<00:00,  7.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(6146.4854, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(6319.8345, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(6181.1553, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2545.9828\n",
      "Epoch  43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:55<00:00,  8.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2000.7216, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2334.7493, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2067.5271, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2543.6803\n",
      "Epoch  44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  8.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1669.0891, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1278.7218, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1591.0157, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.0302\n",
      "Epoch  45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:56<00:00,  7.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1739.2161, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1354.4020, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1662.2533, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2541.7149\n",
      "Epoch  46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [01:58<00:00,  7.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(1531.4813, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1439.7485, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(1513.1348, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2541.5460\n",
      "Epoch  47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:33<00:00,  6.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(6411.7554, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(5260.9507, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(6181.5947, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2542.7772\n",
      "Epoch  48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:42<00:00,  5.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(2297.8765, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(2276.0295, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2293.5071, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "val Loss: 2544.2747\n",
      "Epoch  49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 931/931 [02:42<00:00,  5.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hard_loss:  tensor(3065.8794, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "distillation loss:  tensor(1977.1700, device='cuda:0', grad_fn=<MseLossBackward0>)\n",
      "calculated loss tensor(2848.1377, device='cuda:0', grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "\n",
    "# Specify the path to the saved model checkpoint\n",
    "teacher_checkpoint = 'result/teacher/teacher.pth'\n",
    "teacher = nn.Sequential(\n",
    "    mobilenet,\n",
    "    nn.AdaptiveAvgPool2d(1),  \n",
    "    nn.Flatten(),\n",
    "    LinearRegressionHead(in_features)\n",
    ")\n",
    "\n",
    "# Load the saved model state_dict\n",
    "teacher.load_state_dict(torch.load(teacher_checkpoint))\n",
    "teacher.to(device)\n",
    "teacher.eval()\n",
    "\n",
    "# Set the distillation parameters (alpha, temperature)\n",
    "alpha = 0.8  # Weight for hard targets\n",
    "temperature = 10  # Sharpness of soft targets\n",
    "# KD MSE WITH TEMP 10\n",
    "from tqdm import tqdm\n",
    "for epoch in range(num_epochs):\n",
    "    scheduler.step()\n",
    "    print('Epoch ',epoch)\n",
    "    for i, (inputs, targets) in enumerate(tqdm(train_loader)):\n",
    "        # Zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "        inputs = inputs.to(device)\n",
    "        targets = targets.to(device)\n",
    "\n",
    "        ########### Distillation Loss ###########\n",
    "        # Forward pass\n",
    "        outputs_student = student(inputs)\n",
    "        with torch.no_grad():\n",
    "            outputs_teacher = teacher(inputs)\n",
    "\n",
    "        # Calculate classification loss\n",
    "        hard_loss = criterion(outputs_student[:, 0], targets.float())\n",
    "\n",
    "        soft_student_outputs = F.sigmoid(outputs_student/temperature)   \n",
    "        soft_teacher_outputs = F.sigmoid(outputs_teacher/temperature)\n",
    "        distillation_loss = criterion(outputs_student, outputs_teacher)\n",
    "        # Total loss\n",
    "        loss = alpha * hard_loss + (1 - alpha) * distillation_loss\n",
    "        \n",
    "        # Backward pass and optimization\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    print('hard_loss: ', hard_loss)\n",
    "    print('distillation loss: ', distillation_loss)\n",
    "    print('calculated loss', loss)\n",
    "    # Evaluate the model on the test data\n",
    "    with torch.no_grad():\n",
    "        total_loss = 0\n",
    "        for inputs, targets in val_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            targets = targets.to(device)\n",
    "            outputs = student(inputs)\n",
    "            loss = criterion(outputs[:,0], targets.float())\n",
    "            total_loss += loss.item()\n",
    "        mean_loss = total_loss / len(val_loader)\n",
    "        print(f'val Loss: {mean_loss:.4f}')\n",
    "        losses.append(mean_loss)\n",
    "        if  mean_loss<lowest_loss:\n",
    "            print('Mean_loss '+str(mean_loss)+'lowest_loss'+str(lowest_loss))\n",
    "            print('RMSE LOSS: '+str(torch.sqrt(torch.tensor(mean_loss))))\n",
    "            lowest_loss = mean_loss\n",
    "            torch.save(student.state_dict(), os.path.join(result_dir,'kd_model.pth'))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the losses per epoch\n",
    "import pickle\n",
    "\n",
    "# Save the list to a file\n",
    "with open(result_dir+'/losses.pkl', 'wb') as file:\n",
    "    pickle.dump(losses, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5125.025346427128, 3783.5929223422345, 3180.622602791622, 2934.1552860654633, 2682.950215306775, 2804.6780453385977, 2570.101334144329, 2580.789471330314, 2553.529848822232, 2544.90777219575, 2542.0412502946524, 2551.683973641231, 2542.207899422481, 2548.5630119586813, 2540.95843979408, 2544.0871192669047, 2542.98259077401, 2540.678268958782, 2546.91947042531, 2545.6182114173625, 2540.4398382778827, 2546.107275074926, 2544.701911137022, 2548.604067046067, 2542.0297683189656, 2543.206382488382, 2541.7154630463697, 2544.921569034971, 2543.365880505792, 2544.661238045528, 2544.6026700776197, 2544.0797355914938, 2546.034806350182, 2544.4788402688914, 2542.1497013486664, 2542.3265354551118, 2540.482212987439, 2544.016759412042, 2544.032019253435, 2540.46507000101, 2543.471394505994, 2542.1931073418978, 2545.98276072535, 2543.6802678601493, 2542.030156628839, 2541.7148569041283, 2541.5460294526197, 2542.7771501212283, 2544.274728709254, 2541.300435559503]\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "# Load the list from the file\n",
    "losses_path = os.path.join(result_dir,'losses.pkl')\n",
    "with open(losses_path, 'rb') as file:\n",
    "    losses = pickle.load(file)\n",
    "print(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.system(\"shutdown /s /t 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saveLossDiagram(num_epochs,losses,result_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the path to the saved model checkpoint\n",
    "model_checkpoint_path = 'result\\kd\\kd_model.pth'\n",
    "# Load the saved model state_dict\n",
    "student.load_state_dict(torch.load(model_checkpoint_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load model and get results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size 50 , Epochs 50 , Learning Rate 0.010000 , Loss 10000000.000000\n",
      "MSE Loss: 49.9668\n",
      "result/kd\\results.csv\n"
     ]
    }
   ],
   "source": [
    "outputss = []\n",
    "targetss = []\n",
    "\n",
    "# choose loss to evaluate with \n",
    "if loss == 0:\n",
    "    criterion = nn.MSELoss()\n",
    "else:\n",
    "    def RMSELoss(yhat,y):\n",
    "        return torch.sqrt(torch.mean((yhat-y)**2))\n",
    "    criterion = RMSELoss\n",
    "\n",
    "# Evaluate the model on the validation data\n",
    "with torch.no_grad():\n",
    "    total_loss = 0\n",
    "    for inputs, targets in val_loader:\n",
    "        inputs = inputs.to(device)\n",
    "        targets = targets.to(device)\n",
    "        outputs = student(inputs)\n",
    "        outputss.append(outputs[:,0].tolist())\n",
    "        targetss.append(targets.tolist())\n",
    "        loss = criterion(outputs[:,0], targets.float())\n",
    "        total_loss += loss.item()\n",
    "    mean_loss = total_loss / len(val_loader)\n",
    "    if loss == 0:\n",
    "        print(\"Batch Size %d , Epochs %d , Learning Rate %f , Loss %f\"%(batch_size, num_epochs, lr, lowest_loss))\n",
    "        print(f'MSE Loss: {mean_loss:.4f}')\n",
    "        SaveResults(result_dir, batch_size, num_epochs, lr, mean_loss, lowest_loss)\n",
    "    else:\n",
    "        print(\"Batch Size %d , Epochs %d , Learning Rate %f , Loss %f\"%(batch_size, num_epochs, lr, lowest_loss))\n",
    "        print(f'MSE Loss: {mean_loss:.4f}')\n",
    "        SaveResults(result_dir, batch_size, num_epochs, lr,lowest_loss, mean_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Flatten the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = []\n",
    "for i in outputss:\n",
    "    for j in i:\n",
    "        outputs.append(j)\n",
    "\n",
    "targets = []\n",
    "for i in targetss:\n",
    "    for j in i:\n",
    "        targets.append(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "observedvsPredicted(result_dir, targets, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train and Shut Down"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.system(\"shutdown /s /t 1\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}