Lab07_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IeYtZStL1_UC"
   },
   "source": [
    "# Lab 7-1: Tips"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mDt8Ekbp1_UI"
   },
   "source": [
    "Author: Seungjae Lee (이승재)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4NyWlgO-1_UJ"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7pK3ojFD1_UM",
    "outputId": "1345dfe9-d0ca-4314-efd6-1d9cac50ab66"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7ff0ec6d8e90>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For reproducibility\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ndAeQykM1_UO"
   },
   "source": [
    "## Training and Test Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5MUleULn1_UO"
   },
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[1, 2, 1],\n",
    "                             [1, 3, 2],\n",
    "                             [1, 3, 4],\n",
    "                             [1, 5, 5],\n",
    "                             [1, 7, 5],\n",
    "                             [1, 2, 5],\n",
    "                             [1, 6, 6],\n",
    "                             [1, 7, 7]\n",
    "                            ])\n",
    "y_train = torch.LongTensor([2, 2, 2, 1, 1, 1, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "enCJPkKE1_UP"
   },
   "outputs": [],
   "source": [
    "x_test = torch.FloatTensor([[2, 1, 1], [3, 1, 2], [3, 3, 4]])\n",
    "y_test = torch.LongTensor([2, 2, 2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LLFJ3F3R1_UQ"
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XgQaGBOm1_UR"
   },
   "outputs": [],
   "source": [
    "class SoftmaxClassifierModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(4, 2) #x_train의 shape과 y_train의 class의 갯수를 고려해보세요.\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k504gsZv1_US"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel() #SoftmaxClassifierModel()로 모델을 초기화하세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tDSegsws1_US"
   },
   "outputs": [],
   "source": [
    "# optimizer 설정\n",
    "optimizer =  optim.SGD(model.parameters(), lr=0.1) #SGD optimizer를 사용하고 learning rate는 0.1로 적용하세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FpJEADyk1_UT"
   },
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction =model.(x_train) # #모델을 생성해서 예측값을 구해보세요.\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.cross_entropy(prediction, y_train) #PyTorch에서 제공하는 F.cross_entropy함수를 사용해 cost를 구현해보세요.\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad() # gradient를 0으로 초기화\n",
    "        cost.backward()  # 비용 함수를 미분하여 gradient 계산\n",
    "        optimizer.step()  # W와 b를 업데이트\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MzwtgZPU1_UU"
   },
   "outputs": [],
   "source": [
    "def test(model, optimizer, x_test, y_test):\n",
    "    prediction = model(x_test)\n",
    "    predicted_classes = prediction.max(1)[1] # 각각의 데이터 마다 가장 큰 값의 인덱스를 취함 => class를 예측하는 것\n",
    "    correct_count = (predicted_classes == y_test).sum().item()\n",
    "    cost = F.cross_entropy(prediction, y_test)\n",
    "\n",
    "    print('Accuracy: {}% Cost: {:.6f}'.format(\n",
    "         correct_count / len(y_test) * 100, cost.item()\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AOjN96fz1_UU",
    "outputId": "3df05551-23c2-4f11-eb95-2fa51cc04417"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 2.203667\n",
      "Epoch    1/20 Cost: 1.199645\n",
      "Epoch    2/20 Cost: 1.142985\n",
      "Epoch    3/20 Cost: 1.117769\n",
      "Epoch    4/20 Cost: 1.100901\n",
      "Epoch    5/20 Cost: 1.089523\n",
      "Epoch    6/20 Cost: 1.079872\n",
      "Epoch    7/20 Cost: 1.071320\n",
      "Epoch    8/20 Cost: 1.063325\n",
      "Epoch    9/20 Cost: 1.055720\n",
      "Epoch   10/20 Cost: 1.048378\n",
      "Epoch   11/20 Cost: 1.041245\n",
      "Epoch   12/20 Cost: 1.034285\n",
      "Epoch   13/20 Cost: 1.027478\n",
      "Epoch   14/20 Cost: 1.020813\n",
      "Epoch   15/20 Cost: 1.014279\n",
      "Epoch   16/20 Cost: 1.007872\n",
      "Epoch   17/20 Cost: 1.001586\n",
      "Epoch   18/20 Cost: 0.995419\n",
      "Epoch   19/20 Cost: 0.989365\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NfzfnSRc1_UV",
    "outputId": "aa1c602b-65e3-4a26-ac25-feb38e414d0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.0% Cost: 1.425844\n"
     ]
    }
   ],
   "source": [
    "test(model, optimizer, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9qLNy-E41_UV"
   },
   "source": [
    "## Learning Rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "genGgcdk1_UW"
   },
   "source": [
    "Gradient Descent 에서의 $\\alpha$ 값"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xGzjnTkM1_UW"
   },
   "source": [
    "`optimizer = optim.SGD(model.parameters(), lr=0.1)` 에서 `lr=0.1` 이다"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QVpaEQFNBLNj"
   },
   "source": [
    "Q1. Learning Rate가 다음과 같을 때 어떤 일이 발생할까요?\n",
    "\n",
    "Learing rate가 클 경우 : 큰 오차가 생길 수 있다.\n",
    "\n",
    "Learing rate가 작을 경우 : 결과를 도출해내는 데에 너무 오래 걸릴 수 있다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C5z1kNJy1_UW"
   },
   "source": [
    "Large learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CHsBXQFT1_UW"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hhqLUXTL1_UX"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EbzgO5nq1_UX",
    "outputId": "5a600459-81c1-49b3-dfc5-dbb7a6c0b964"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 1.280268\n",
      "Epoch    1/20 Cost: 976950.812500\n",
      "Epoch    2/20 Cost: 1279135.125000\n",
      "Epoch    3/20 Cost: 1198379.000000\n",
      "Epoch    4/20 Cost: 1098825.875000\n",
      "Epoch    5/20 Cost: 1968197.625000\n",
      "Epoch    6/20 Cost: 284763.250000\n",
      "Epoch    7/20 Cost: 1532260.125000\n",
      "Epoch    8/20 Cost: 1651504.000000\n",
      "Epoch    9/20 Cost: 521878.500000\n",
      "Epoch   10/20 Cost: 1397263.250000\n",
      "Epoch   11/20 Cost: 750986.250000\n",
      "Epoch   12/20 Cost: 918691.500000\n",
      "Epoch   13/20 Cost: 1487888.250000\n",
      "Epoch   14/20 Cost: 1582260.125000\n",
      "Epoch   15/20 Cost: 685818.062500\n",
      "Epoch   16/20 Cost: 1140048.750000\n",
      "Epoch   17/20 Cost: 940566.500000\n",
      "Epoch   18/20 Cost: 931638.250000\n",
      "Epoch   19/20 Cost: 1971322.625000\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Wc12FsyN1_UX"
   },
   "source": [
    "Small learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "URf6c-0i1_UY"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "y6nPVEGV1_UY"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ARubAn4x1_UY",
    "outputId": "980be48a-ad72-43cb-b027-4ce370d9fdba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 3.187324\n",
      "Epoch    1/20 Cost: 3.187324\n",
      "Epoch    2/20 Cost: 3.187324\n",
      "Epoch    3/20 Cost: 3.187324\n",
      "Epoch    4/20 Cost: 3.187324\n",
      "Epoch    5/20 Cost: 3.187324\n",
      "Epoch    6/20 Cost: 3.187324\n",
      "Epoch    7/20 Cost: 3.187324\n",
      "Epoch    8/20 Cost: 3.187324\n",
      "Epoch    9/20 Cost: 3.187324\n",
      "Epoch   10/20 Cost: 3.187324\n",
      "Epoch   11/20 Cost: 3.187324\n",
      "Epoch   12/20 Cost: 3.187324\n",
      "Epoch   13/20 Cost: 3.187324\n",
      "Epoch   14/20 Cost: 3.187324\n",
      "Epoch   15/20 Cost: 3.187324\n",
      "Epoch   16/20 Cost: 3.187324\n",
      "Epoch   17/20 Cost: 3.187324\n",
      "Epoch   18/20 Cost: 3.187324\n",
      "Epoch   19/20 Cost: 3.187324\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AAfx42ne1_UZ"
   },
   "source": [
    "적절한 숫자로 시작해 발산하면 작게, cost가 줄어들지 않으면 크게 조정하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dAb7ZOKw1_UZ"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "V069kTVp1_UZ"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Aq9HM-Z_1_UZ",
    "outputId": "a6df5cf4-9acd-4d4e-a0a0-6488a2b18a26"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 1.341573\n",
      "Epoch    1/20 Cost: 1.198802\n",
      "Epoch    2/20 Cost: 1.150877\n",
      "Epoch    3/20 Cost: 1.131977\n",
      "Epoch    4/20 Cost: 1.116242\n",
      "Epoch    5/20 Cost: 1.102514\n",
      "Epoch    6/20 Cost: 1.089676\n",
      "Epoch    7/20 Cost: 1.077479\n",
      "Epoch    8/20 Cost: 1.065775\n",
      "Epoch    9/20 Cost: 1.054511\n",
      "Epoch   10/20 Cost: 1.043655\n",
      "Epoch   11/20 Cost: 1.033187\n",
      "Epoch   12/20 Cost: 1.023091\n",
      "Epoch   13/20 Cost: 1.013356\n",
      "Epoch   14/20 Cost: 1.003968\n",
      "Epoch   15/20 Cost: 0.994917\n",
      "Epoch   16/20 Cost: 0.986189\n",
      "Epoch   17/20 Cost: 0.977775\n",
      "Epoch   18/20 Cost: 0.969660\n",
      "Epoch   19/20 Cost: 0.961836\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FPSYQ-lH1_Ua"
   },
   "source": [
    "## Data Preprocessing (데이터 전처리)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "551p9eWA1_Ua"
   },
   "source": [
    "데이터를 zero-center하고 normalize하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eVBCPsDD1_Ua"
   },
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[73, 80, 75],\n",
    "                             [93, 88, 93],\n",
    "                             [89, 91, 90],\n",
    "                             [96, 98, 100],\n",
    "                             [73, 66, 70]])\n",
    "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "17vDm7i01_Ub"
   },
   "source": [
    "$$ x'_j = \\frac{x_j - \\mu_j}{\\sigma_j} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y7shP1Jt1_Uc"
   },
   "source": [
    "여기서 $\\sigma$ 는 standard deviation, $\\mu$ 는 평균값 이다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "r0pu6TcI1_Uc"
   },
   "outputs": [],
   "source": [
    "mu = x_train.mean(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "opFSudRu1_Ue"
   },
   "outputs": [],
   "source": [
    "sigma = x_train.std(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2OEH3LeZ1_Ue"
   },
   "outputs": [],
   "source": [
    "norm_x_train = (x_train - mu) / sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AOuJijWj1_Ue",
    "outputId": "eec2d117-d97b-485e-d3b2-e19d5a45dd05"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0674, -0.3758, -0.8398],\n",
      "        [ 0.7418,  0.2778,  0.5863],\n",
      "        [ 0.3799,  0.5229,  0.3486],\n",
      "        [ 1.0132,  1.0948,  1.1409],\n",
      "        [-1.0674, -1.5197, -1.2360]])\n"
     ]
    }
   ],
   "source": [
    "print(norm_x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ig-JfKxp1_Uf"
   },
   "source": [
    "Normalize와 zero center한 X로 학습해서 성능을 보자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UJwCzulq1_Uf"
   },
   "outputs": [],
   "source": [
    "class MultivariateLinearRegressionModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(3, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eRBrSASh1_Uf"
   },
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hs0e8drb1_Uf"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pWZUOQgB1_Ug"
   },
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7eAnbGrj1_Ug",
    "outputId": "255440df-c35b-4035-8a88-5e8b71a85df0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 29785.091797\n",
      "Epoch    1/20 Cost: 18906.164062\n",
      "Epoch    2/20 Cost: 12054.674805\n",
      "Epoch    3/20 Cost: 7702.029297\n",
      "Epoch    4/20 Cost: 4925.733398\n",
      "Epoch    5/20 Cost: 3151.632568\n",
      "Epoch    6/20 Cost: 2016.996094\n",
      "Epoch    7/20 Cost: 1291.051270\n",
      "Epoch    8/20 Cost: 826.505310\n",
      "Epoch    9/20 Cost: 529.207336\n",
      "Epoch   10/20 Cost: 338.934204\n",
      "Epoch   11/20 Cost: 217.153549\n",
      "Epoch   12/20 Cost: 139.206741\n",
      "Epoch   13/20 Cost: 89.313782\n",
      "Epoch   14/20 Cost: 57.375462\n",
      "Epoch   15/20 Cost: 36.928429\n",
      "Epoch   16/20 Cost: 23.835772\n",
      "Epoch   17/20 Cost: 15.450428\n",
      "Epoch   18/20 Cost: 10.077808\n",
      "Epoch   19/20 Cost: 6.633700\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer,  y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lMVmpArvAXnz"
   },
   "source": [
    "Q2. 이렇게 Data Preprocessing을 하는 이유가 무엇일까요?\n",
    "\n",
    "=> 기존의 데이터를 알고리즘에 알맞는 데이터로 변환시키고, 모델 학습의 성능을 높이기 위해서"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UyQBI9z61_Ug"
   },
   "source": [
    "## Overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CaKMCCEZCBSD"
   },
   "source": [
    "Q3. Overfitting의 개념은?\n",
    "\n",
    "=> 학습 모델이 주어진 데이터에 너무 과하게 맞추어져서 조금이라도 다른 데이터가 들어와도 다른 결과로 예측하여 결과적으로 정확도가 떨어지는 현상"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bcJLcLhSCBVh"
   },
   "source": [
    "Q4. 그렇다면 Overfitting을 방지하는 방법에는 무엇이 있을까요?\n",
    "\n",
    "=> 1  데이터의 양을 늘린다. : 데이터의 양이 적을 경우, 노이즈까지 암기하여 과적합이 발생할 수 있다.\n",
    "   2  모델적합도 줄이기 : 히든 레이어의 수, 매개변수의 수 등을 줄인다.\n",
    "   3  가중치 규제 : L1 규제 : 가중치 w들의 절대값 합계를 비용 함수에 추가합니다. L1 노름이라고도 합니다.\n",
    "                    L2 규제 : 모든 가중치 w들의 제곱합을 비용 함수에 추가합니다. L2 노름이라고도 합니다.\n",
    "   4  드롭 아웃 : 신경망의 일부를 쓰지 않는다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RwbeaBBG1_Uh"
   },
   "source": [
    "Regularization: Let's not have too big numbers in the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XCdG_T_l1_Uh"
   },
   "outputs": [],
   "source": [
    "def train_with_regularization(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "        \n",
    "        # l2 norm 계산\n",
    "        l2_reg = 0\n",
    "        for param in model.parameters():\n",
    "            l2_reg += torch.norm(param)\n",
    "            \n",
    "        cost += l2_reg\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch+1, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uTw_T6D11_Uh"
   },
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YzVc7CUb1_Uh"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZafKSN8N1_Ui",
    "outputId": "d53951cb-5c8d-4c0f-a8a2-e80db172d312"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    1/20 Cost: 29477.810547\n",
      "Epoch    2/20 Cost: 18798.513672\n",
      "Epoch    3/20 Cost: 12059.364258\n",
      "Epoch    4/20 Cost: 7773.400879\n",
      "Epoch    5/20 Cost: 5038.263672\n",
      "Epoch    6/20 Cost: 3290.066406\n",
      "Epoch    7/20 Cost: 2171.882568\n",
      "Epoch    8/20 Cost: 1456.434570\n",
      "Epoch    9/20 Cost: 998.598267\n",
      "Epoch   10/20 Cost: 705.595398\n",
      "Epoch   11/20 Cost: 518.073608\n",
      "Epoch   12/20 Cost: 398.057220\n",
      "Epoch   13/20 Cost: 321.242920\n",
      "Epoch   14/20 Cost: 272.078247\n",
      "Epoch   15/20 Cost: 240.609131\n",
      "Epoch   16/20 Cost: 220.465637\n",
      "Epoch   17/20 Cost: 207.570602\n",
      "Epoch   18/20 Cost: 199.314804\n",
      "Epoch   19/20 Cost: 194.028214\n",
      "Epoch   20/20 Cost: 190.642014\n"
     ]
    }
   ],
   "source": [
    "train_with_regularization(model, optimizer, norm_x_train, y_train)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Lab07_1.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}