RNN - air pollution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN - Air Pollution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tensorflow.keras.layers import SimpleRNN, LSTM, GRU\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import optimizers, metrics\n",
    "from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/air_pollution.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.set_index('No', inplace=True)\n",
    "df = df[['pm2.5', 'TEMP', 'DEWP', 'PRES', 'Ir', 'Iws', 'Is']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='No'>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEGCAYAAACAd+UpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7d0lEQVR4nO2dd3wVZfb/P4d0EkhCEkJIgBB6lRKKiKKASlGxl3Vtuy679vb9uVh2ddeyWNZ1dW2sjd21rKtrWQMiTQWkiEiVDqGXUALBENKe3x93Jpnc3DJzp8+c9+uVV+59Zu7MmXbmPOc5zzkkhADDMAzjTVrYLQDDMAxjHqzkGYZhPAwreYZhGA/DSp5hGMbDsJJnGIbxMPF2C6AkOztbFBYW2i0GwzCMq/j+++8PCSFyQi3TreSJKBnANwCSpO19KIR4hIg6A3gfQBaA7wFcJ4SojrStwsJCLF++XK9IDMMwvoKIdoRbZoS75hSA0UKI0wAMADCOiIYDeArAX4QQXQEcBfBLA/bFMAzDaEC3khcBTkhfE6Q/AWA0gA+l9ukALta7L4ZhGEYbhgy8ElEcEa0EcBDAbABbAZQLIWqlVXYDyA/z28lEtJyIlpeVlRkhDsMwDCNhiJIXQtQJIQYAKAAwFEBPDb+dJoQoFkIU5+SEHDdgGIZhYsTQEEohRDmA+QBOB5BBRPLAbgGAPUbui2EYhomObiVPRDlElCF9TgFwLoD1CCj7y6XVbgDwqd59MQzDMNowIk4+D8B0IopD4KXxgRDicyL6EcD7RPQ4gB8AvGHAvhiGYRgNGBFds1oIMVAI0V8I0VcI8UepfZsQYqgQoqsQ4gohxCn94jqfk9V1uOmtZTh2ssZuUSxh84EK3P3+D3aLwTBMGDitgcFc/+ZSzN9YhlHPzLdbFEs49y/f4JOVe7G17ET0lRmGsRxW8gazcX8FAKC80h+WvExtHRefYRgnwkreYGp8quxu/sd32Ft+0m4xLOOHnUcx+8cDmLlmH9bsPma3OAwTFkclKPMCNXX1dotgC7uOnMT1by7DnHtH2S2KJVzy8rdNvpdOnWiTJAwTGbbkDaa23l+WfFpSo53gNxcVw7gBtuRNIjHOH+9PZSH4QydO4eppi3HvuT0wtHMbG6VizKK8shovzN2Co5XV+OlULaZdX2y3SEwUWMmbRLVP3DbB/ZYl247gytcWs/vCozw5Yz0+WL7bbjEYDfjD3LSBjJYJdotgCfXCX+4pv1Nd6w/jxUuwkjeQGWv2NXyOI8Lf5m3Gpyu9nbKnqib0Qy+EwP/9ZxV+2HnUYomMp6qmDte9sRSFU0rQ/aGZuPeDlXaLZAm7j1bi1/9cjpPVdQ1tPhtywtGfqlE4pQQLNx+yW5SYYSVvILe+s6Lhc6vkeDz75Sbc9f5K+wSykfLKGnz4/W7c9PZ3douimwWbD2GB9JBX19Xjvyu8/eKWeXLGesxadwDzNhxsaKvzWc/tiRnrAQA/f2OpzZLEDit5k2hBZLcIjEHU+c18jYDwmZL3wrXngVeD+PnrTd/02w791PB5weYynNnNG7nyj/xUjUGPzcZdY7rhi7X7w64nh5LWe+AhUTPuUDilBN1z07DpwAmM69MOr1432ALJzON/q/ZixprA9b3t3RVIjC/Gub1zG9q8iBACnR+YgfyMFOyRJvZ1ympps1T6YUveIBZuCe+z+90nay2UxFz+t2ovAOCvczdj44GKsOtV1QT8uJUKf65bUWvNbToQyN/zxTr3K8I73muadO4+H4xDnJIGlfcoZm7vOFxplziGwUreAryg6GRO1ao7Ftlb1bZVkonSWANHEAFxLbzvfvTqdWZ3jQUcrPBGluU//G8d3lpUqmrd7ZK7au+xKhMlMp9fvv0d5ioGHv3K0coaFE4psVsMU1GTd+rBj9dg1a5ylNx5pgUSGQMreUY1ahU8AM9MmGEF7x8i9VIv6J8HAHh36U6rxDEMdtdYSF298E10gl+Ok/EOVdXhJ3qlJMQ1+e6mqBtW8jr46VQtCqeUqOrGFk4pQZcHZ+D+D1dbIJn9fL56X/SVGEey+6j6wUYvuXAe+Dj8sxnXgjD0iTkN37s8OANfbyqzQizdsJLXwYHj2v3N//neG24MtZzZLdtuERiNrPZpfvxFWw6HXZaZmthsbO0/y3eZLZIhsJKPgaM/VePA8SqcrIktauanU7UGS2Q+VTEea1wLQq3LkrVV1dTpltmN11hGbQSVTKz3hlOorK6NOp+joqp5Gu0jP1WbJZKh6FbyRNSBiOYT0Y9EtI6I7pLa2xDRbCLaLP3P1C+uMxj42GwMe3IuJr6wMKbf93lkFpZtP2KwVObS83dfxPS7rzaWYfI/vzdYGnPp+bsvcOVrixsihGKhzyOzsN+lkUX3/HuVpvVjvTecQHVtPXr/fhaKHpwRcb1/LWk+4Prt1sMocYFb0ghLvhbAfUKI3gCGA7iNiHoDmAJgrhCiG4C50ndGYuUu9yfuUss8F0aorNhZji0H9RUn33vMP+UQ3YreAdQ56w8YJIl56FbyQoh9QogV0ucKAOsB5AOYBGC6tNp0ABfr3ZeXcFNum11H9M/6i2X8wm4qq/W5XNxzhRvZWqbvxeY2WujUgAcrnH9fG+qTJ6JCAAMBLAWQK4SQ+zL7AeSG+c1kIlpORMvLytwxWm0EpYdjdwVYyapd5Tjz6fm6tzPsybkGSGMtejOIuiX6QsmYP39ttwiW8sLczbp+v2jLYceHCxum5IkoDcBHAO4WQhxXLhOBsxDyTAghpgkhioUQxTk53kjipYaaWmffGDJqXkbXDutogSTuY1OE3D6MM5i1Tr+7xekh84bMeCWiBAQU/DtCiP9KzQeIKE8IsY+I8gC4zzFrIl7Kk1GYlWq3CLqpravHur3HcVqHDMO2meCiOr/zNx7Emwu3x/z7nYcrMXv9AVTV1OHGEYVITXLuZPqN+yuQ2TIB1XX1usddAKC2vh5xLeKir2gTuq8EERGANwCsF0I8p1j0GYAbAEyV/n+qd19OoLwycthUYnwLVSXS2mekGCWSqagZOxhc6P7Aqedmb8LLX21FyZ0jDdumW5T8vA0H8Iu3l+vaxlnPNLr0npm10dE1fs9//hv0L0g3bD5AVXU9kuKdq+SNuAvPAHAdgNFEtFL6m4CAcj+XiDYDGCt9dz2RYuP/cFEfLHtwjKrttG3tjuyMapT8oI7uV/Jr9gQe+DKNyeQuGZgfdllqonMffCWlh9yfTlcrRk74qql39jwQ3Za8EGIhwgcSqNN4DqT00E84WVOHXnmtAQDbyk6guq4eK3eWh/1NZmqi6pSstSoy3tnFweNV2HX0JDYdqMBXG/3hZZO9Z8ertEXUJCeEt5MOnajG9G9LMaZXWxRkOrP4xIqdRzF9candYliGGRPzznnmK1ScqsWyh8agbatkw7evF+c6zmzm7Ge/AoCGbudoFVEHAwoyVG/fyT7LcX9doHo2X7e2aaq3e7CiypEPAdBY9OXOoGIZ0RjTMxfvLQs9vb1kzT6UrNmHRz5b51j3xaUvf2vKdo+drEF6SoIp29aDGTmVKqTZzUOfmOvI6+xcTeNCOma1DDn9ORRJ8c7112qZrj373lGq142U5c+NlE6dGHWMxq84NZWFluRrXsG5msbj1AuBtXuOYcm28EmRvMb/Vu/F/R+u8kTdV8adrNxVbrcIlsOWvMGoVV/1QuCCFwO5b5zYxYuVSNFFz8zaCAC4orgDhhS2sVIshgEAzFnvjzEmJWzJ24TDB+Rj5u6x3QAAvx5VFHYdNSGmTqJVcnhbiFyZvIDxE6zkbeK+/zRm+rvxrWWqfflOR43S21Z2wvYyam8u3I5b/hXIjvnoZ+sirluhMeLG6bw0fwtunv6dqft49eutrsxXpJeZa/ZheamzMsyyu8YBfLWxDFNnbsATl/SzWxRL+N2nAaX6MxvTIfzx8x8bPr/9baltctiB7DYzk78v2I7lO47i41vPMH1fTuKWd1YAcJYLli15hxBrARLGXoTqURj/ccJjPSC3wko+CstLj+C1r7eavyMf6orvd9ifU7/oAX01StWmILrt3RWY7rAew01vLTN1+8cd4oL8Yu0+fOSzsptKWMlH4fJXF+NPMzeoXr9VjJOcvtvhLD9eLFw6KB8/G9YR5/bOxeQzww+8ylz2ijkTcbSgJprzgfE9m7U9OCHQpvbdXLI6MCnKSczfaG4q5APHtaWIMIvf/GtFkzEwv8E+eYMhFxUDMRKlD/Lv1xfbKInxjOnVFn+auQEZLRNQXhmwTief1QUAHJ9LnGHYkmea8NL8LXaLYDqLpBQG6vHni9sIdjioOI4lbleJ695Yiv6PztJVJ9goWMk7BKeUA7Qi8sJurn19qab1M1oGcrAQgKcu64d3bh7WsMzrdnzn7Ka1Ap64pC/e/dWwMGs3x0mRS1rcrnpZsPkQjlfV4k8z1lu2z3CwkncIarNXMvrR+j5Vrn7VkI44o2t2w3cve2vy0pMx///ObtJ27bBOGNElO/QPQuDkbKt+gZW8QxACKJxSgmtfX2LL/p+ZtQE3T9dXOCIWCqeU4Jxnv7LM4vnVP5ZrVszy6n5LuWNEFsl/LtlhgCSxUVtXj8Ip+qKn9OKEW4aVvAV88OvT0b8gPeI6x08GBvQWbbEnYdlL87dizvrI9S4n9sszZd/bD/2E177ZZsq2g5n9o/aannKpxmMnm4cEejVOPjmhBd66aYjdYugi1PXSwivXDsJnt+ubzOWEDjoreQsY2rkNPrvduLJydpEUoUCGp5H0eHZaiGpe3tTx+Pv1xchLd0eJSrMY3y8P/TXUiAiFE9ywHELpEA5ryOFuF04sAmEFsh4P5cvXWsdV6T5w0tR3LzJi6jy7RcBBB8wV8KlpZi4/H25fThYzmdgvD5cOzMc1QzsAAC7oH9l98/7k4VaIZTqyDz+UTZaZmohXfz7IUnmsQJlo7vw+uTZKEjunNGQ7HWpS6usVO+2f1c1K3gTO0BB9EIwTuneReO6qAejathWAMO4LBcOLsqwQyXRkv3u4qJxxfc0Zq3AKp3vkOkZieJF36xsYouSJ6E0iOkhEaxVtbYhoNhFtlv5nGrEvN6DHTVvnohAOh4T2m05SfBwAoLdU1N0P1BpY8OB3n6yNvpLBPPWFtpj4uBbetXeNOrK3AYwLapsCYK4QohuAudJ3xsXIrx+/TeVvk5qIf08ejhd/5j23TDiUk/Pkq31ah4yYtmVHGOU0DdFa+RkpyGkV6JV60aI3RMkLIb4BEJxhaxKA6dLn6QAuNmJfbsAvOtBPVZGGFWUhLcbkc15hYIxK3g7qNTyEZ3XPbuiVpiUZG1zghI65mX2UXCHEPunzfgCuGb3xY0UbNQQ/N0a7a77dqjWnDKOVT1fusVsES9BiaB0/2Zj3PjlEmLBe6/7z1Xt1/V4vljiiRKB/H/K0E9FkIlpORMvLysxNfaqWveUnIy4PVb+0T/vo/tr/d34PAJFrhvqZlbvK7RbB88xatz/i8japic3avN4zTUuKbzjGVsnxeH/ycHylSOfw5o1D8NaNQ2KOFlu81Z4JjjJmKvkDRJQHANL/kGXShRDThBDFQojinJwcE8VRx9z1B6J2sUZ2zUZhVssmbcoY8nCzIGXl7rQIGq2ZAs166P3k/gFgeYbCg8erotardfPg8vGqGnyxdj/KKrTHpjc+s4ThRVkoVCRma5kYj3N6to05Wszu591Mk/IzADcAmCr9/9TEfRnCfR+swkcrdqMgM/JMv/yMlGZqXKn4oinBSMtDdRfNZN+xkxj1zFeq1pUHXOVey+BOxgZMrd17zNDtOZ1znv0Ks+85C91yW1myv6FPzo3pd72l6z3I4OttNP0f/TKm3xEBPdsFrsHQzsYfo90V0IwKoXwPwGIAPYhoNxH9EgHlfi4RbQYwVvruaGR/5e6jkd01RTlpmgZ2tHBWN2t7M0dimGk7oms2ljwwBhMMzmWz+0ilodszgidNLq6+3wXjP8OLsrDkgTG46LT2DW1/uKiPjRIZixDA4E5tsOSBMbhkYIHh299x2N772hBLXghxTZhFY4zYvpkcr6rBriOV6NM+HS1akOrhcAPDiJtgZdeutq4eK3aWq15feWbapScbLs+xkzVYs/sYinJSkeqQSJYuOanRV9JBnEMnGwSLFXy9taZzWLvnGPrmR07SZzdm3NOA/fNJvDsDQCU3vLkME19YGPiiwTi/ojj8G79XGL/mwA6BrmCk7HgpiXHqhdDJ83M2a5qo0iloHMJoSg9X4sK/LcSt76wwdT9OooXDxmdkjO6oXvDiQsfOr9CihId2DkTaJMarV515Jr081OJ7Jf+DwpJVY0VvfmI8AOCuMd2w6fHxDe3KwdaubdOw8fHguWFAv4L0kO1KYi0EHgsb9h9Xve7Gx8dZlpXQCfk+ZMyu2ZukQVkw9nP54IBx16Zl8yikcHTJSTNLHFXwHSYhhMDJmrqo68ndVCJq8jYPNlLkqfDBhGtXsuXgCazd46xBSDVyG0VNnUm+sBgwu6tttntueekR7IkSEhyKaMcdy3lxqCEfE1rqCNjdW2MlL+GUATABYOxzX+OCFxfaLYptVNU4R8kXZZvskzdZAVz+6mKMfGoejlfpK6ARzLDO2icIOVXHKweU1dKxjXrXpd1uKlbyEiero1vxkTDqMnrJ2vECWVEyberFioF2IYB6g+fXF+WkeSIffunUiRjRVXvWWC21Fcx2+UWDlbzEzLWRZwJahVXl5A5WVGHO+pDz0xgLsSq6xgnGw+Wvfos/f7nRbjEMQsN1s/ncs5KXeGaWzptPw4WMNInIqoRGN731nTU7YiJilb9W7byOW8/uAgDoZ0K44w87y/HivC2Gb9fpDLA5sRsreaPQ8Kz2jZDnxiqL64ADypJ5kSGFzpwVqtZ4GNE1G6VTJyJDQ/QIA3TPDR9BEyofkJWwkncc1mh5uweDrOKNhduxteyE3WKE5cmS9SivNLa+7z8Wl2LB5jL8bd7mhjar3IBMc+yuR8JKXidyojK57qlerNK9fnjka+rq8djnP+KyV74FAFSpCJG1mrkbDuKfi40tqvH7T9fhujeW4dkvNzW0+eSdbhmtk+NRpJgNHen8tuCBV/fw3JWnNWvLbZ3c5L9eLFPyPnrqo2VeNJJYMmmeOGW+fD663JZARE1SDzv59LKS10CoF7LRL2krutWrdpXjaKWxcdNORq6b61RFZ+Tg65sLt4dsH/6n2DJQMqEJfu4jGU1233es5DUQapLOM5efhisGF2BIofrJIQWZ4SdSWHFDTHppkfk7cQDBESV2J4oKh9Z8/pH44+c/GrYtNUw+q3kBHbfQv0B7BNFFp7XHJQPzcf+4Hk3aIz22do+HOCPVn0toGSJ5WIc2LfHMFc3dOJGIpGwcamy6ErstKLW4Rc5QRKu94GRGxjAJKjkhDn+5agCAQBEWNdh9fX1jyVdU1eCSlxdhmxRp8dmqvSicUmKzVM358PvdDZ8n/W2hI2V0C0pLftWucksetuQYsojOXLs/pvwyTsChnSNLaDKTNcK9de8HqxoG/feWn8Sklxbh8AnrQph9o+TnbTiIH3aW4y9zAmFld773g80SRWfVbmclKYuFz+8Yadu+lUr94U/WWtJtfvaK/rhjdFfNv3tjQWhfuuNxqg9MBXrvhpxWjSkvom1rnVT17PUF27FqVzk+/sG6guq+UfLyA+/eW9Kd9M1Px3XDO9myb6UlnxjfwhJLvm2rZNx3Xo/oK3oEh6bDV4WR94PaaDX5nrQyrNI3Sp6xD7sGnpR7/X7HUUePd+h95ie9tAgvzbc+ZUBaDPUPVu0q173ftXuOofvDM3FAR/ZYI+9LNVtauu0w3v62FABQWW1dWK9vlLx8QWXLI9I05LDbcLKWcDBW5eMJRgQFQ6m1tp6XBtb08PZNQzStr9euW7WrXHf+pb9cpS2AAAAu6N8eD0/spek3by7S75p6a1Epqmvr8fWmMt3bMoJot5YQwAuKGcgtE62LefGPkpfdNZLJlMm5OSzDrpdjsKWmVoyLB+br3nekJHShcIJru11r7ZEycS0IN5+pLYzSiMyb8rV1wGkDEL1XEGzoWFnL2XQlT0TjiGgjEW0hoilm7y8cwYomFr1j14OoN/KicEoJnvpig0HSaCeWXOaFU0oa/rRy6cuLUDilBAcrmkYw7DxcqXlbagku26j1If77gu1YvbscAPDJD3tQOKUkYi1gp9GhjfoXxH+NGHSUbinZtx1LPYjWyepzwkdj15HIz+iVry3Goi2HG757RskTURyAlwCMB9AbwDVE1NvMfUaVSf4Qg5a3yyI1ohTgK19tNUCS2FCb5tYoVkh1e4PP25Jth0OsbQyz7jkL79w8rOF7y8R4vHljMdq2Ul905AuppsFbkjtjm4MTqwXz0W9GWLo/+Y6SDa99x7QbQmfEECdvFJ5R8gCGAtgihNgmhKgG8D6ASSbvMyTBasZqxaMHIRqn5ruROp3nOtaqRlZe4vYZKc2UxuieuZoqS8kPflJCINbe6jKIegYi22rM3aT3fpbHV2QlXx1DXWA7XT1WRiWZreTzAexSfN8ttdmHdHLtVJlay4H95l/fo8uDM0ySxnz0FgEvivHYg6/x4yXrdckRC0nx6h8x+a5Ytv0IAOiKHHE6936wUtfvGyx5xO6usZNYEtnFiu0Dr0Q0mYiWE9HysjLzRsob3vzSyXWTJe92stP8O8itJcQw+OVvRXZKu/h05V5dv28MpAj8t1vJL394LKaM76l6/YR47yj5PQCUidYLpLYGhBDThBDFQojinJwc0wRp5sMr966VFA67lIZdriYnpFPW4gIJ9tPWhHFB1NbVNxQaKas4ZekUeTP46VRt1Ljx2rp67D/W+MwGG2knY6gVYOTdkZ2WhN+M6qJ6fS9NhvoOQDci6kxEiQCuBvCZyfsMjeKKLt56GPs93BUOR99HZsX0u6Ls1OgrRUCvuyZWth0yLrujFRwNqhB1qja0kr//o9UY8MfZqKsXGPLEHAx+fI4xAtj0TuzzyKyo9+bjJesx/E9zG3LANBptAWVpZc0AI7BSyZsakS+EqCWi2wHMAhAH4E0hxDoz9xkNgjHRKmpY/vBYS/ZjJoumjEbrZH23SWpSQMkPKczEd6VHjRBLFW7zaR8/2VRRhatkJec9cUJPxSiidfbeXbYTQGCANTkhrkHLy6oyXK/HqXhGyQOAEGIGAEtHDT9fvRfj++aFDFPadPCEpogHPWRbtJ9wbDlYgW1l+qzZ/Az9qWRlXWSFRb/7qHmx8LGgZYAtMWiQNpwlL5/PcMs9SdBLYMeRwH0t68qaOu0vPDtfklZG13gun/y0b7biyRkbUJC5AQt/O7qhXfbZrdpVHnPujK5ttadCsJOxz31jtwgAgH5ScYbiwkws3HLI1H2NfGp+w+eObcIXZ4lG6+R4HDfABaDFJx8cidMhQnEZIBB15Tdkvbx2z3EAjRZxu3TtBlW7dGNKdiprvarFyGpg0fCckl+wOaBEdh9tOjlC76Djeb1z0TdfeyUZBhhelIXlD4/FweOn8PyczdF/YBDd2raK+bdLHxxreQSW3PPMTkvCoROnohoV8r1uFHqPds2j5+HpLzbin0uMLUweCVlV5meof6Fvenw89pSfRF66/l7qmkfPQ0Kc9qFNLw28OobtOgfhcjTMXIyG8vIGT4c3gp2HK1Ea5njlGGyrMdp1tX7f8ahl8/Qo6ZTEOKSacG0isfNIwNV0SIqWqQ5yx5yqrcNSE2ft6qVVcgKSE8xXKcrJcev3BSz6jQcqVP8+Mb4FOusMJpAJHLN2N+QPO49alonSc0q+n2RtpwSdeGXFpVgY2ytX1++VDO3cWA92bG/jtitz1jPzcfazX4VcduVriw3fnx1MfGEBfvb3pRHXccJcCC0izP7xQJPvT89qmm/oiZL1uGraEiPECokRtqVZFqrS7aUsuPHCvEB6ZTcUAVLy8ldbce+/V1myL88pefkNPb5fO8O2ue3JCTinZ1vDttc3Px0T++cBAM7ukYOSO+2rnuRW6kX0xG1qFWwsPlUr2Hygae6aTRqs1Vgw4pWodTa3ZgSwy2GD67Hyo9QLMRvPKfkqqYsrT8D5x+JSlKzep2ubZg6SmP5Q+Bi1ucat9I9GQxnxoZzgU1VThyXb7HG1aUHLqfxg+a6wmTaXbjuMnYcr8fnqvThZXRcxeuaef6/UKKUzkN1zZuO5gdf3lgbiaUtW78Nfrx6I339qa1h+WK4e0gElq/dhcKdMHP2pOvoPPECBIh3tiC5ZWLPnmKmTWGKpo2m0vtfqMVq8tanPva5eIK4FYepM81NF92gX+0C1zOlFWaoznt7/4eoGn3owSrfUNUM7NFkWfE6trJcajbN75OCrjc4oZCLjOUtejpOudXjWxjO75aB06kTkZ6SoVgRun/yizN/97q+G47PbneGmUhax2P6niTZKgrBhm7Gk0tWKEYPjnbK0ha0Gu6RCsVeRgkRAOPo5GNXdvNQsseI5Je/g6x8WtQOEQgS67X+bt9lVBSXCEYvRvHF/BSa+sMBYOUz01mjd9vvf7WzyXVZoO0wseGInofIa1UaZvVrh4MRtznH8NeI5JT9pYHsAwNhebXGq1h3pR7W8l2au3Ydnv9yEtxeVmiWOqVxZXIBLpfJ6SgV4Tg91FtD4v36DdXuNHbBqnWJchaBgtBodwV19+ecb9ps76GoUWsc3FocICX3/u11Nvgefwrdceu/bhed88vIsx87ZqaiNYaqzHajtfgo0JmI65NLMg09f3lgsWjnlf0SXbMxX4cs0wwuXGMNkFqtwY89UL5Hix51+PpwYSOHcuztGZCXQgsg1uT3U3reXvvItXvt6m6myWImW56G2rh5/mmlO0Q8HPpcNPP3FBte+0I3iG0WU1MDHZtsoifE8/cUGlFWYe309qOQb09N9uW6/vcKopHdeawwvahN1vVW7yhtiw52smGLhssEFEZfPWnfAtBecXLZPzTXQijyJ5+GJvWL6/esLt+Phj9caKZKpKO/LqZf2i20bjvRsm8PLX23FA/9dbeo+PKfkhcKSj6Xuox0kJ8Th/cmn2y2GbRRkpqBNauTqUbX15l3L1MTA7OguOeYloOvdvnXMv3XLfQw0uivy0pNx9dCONkvjDsz2OHjCJy+EwBMl6zGxf16Df7umtt6xMfJGEM43uWCzs2J01aCnV/LR97tRWVOHjm1aYuXOcl0CONXd6ya7Vp43qMd3/to33nFJqmHB5kN49eutmipLacETlrwQgW7tJS9/2+CTn7P+QOQfuZxwedOve2NZTNvrkdt8IsxHt4yIaVtqKchMwS9HdsZbNw6JeRv3/WcVfvfJWtzw5jL8Zc6mmLYhK1EzBvWU27z9nK4xbcNNrjnZ1aIlxXIwRo5BXHha+ybfH72wt2HbNhIzJ7t5Q8krPss++VAFQ7yE0aP4t41uqoBKp07E4E6Zhu4jGCLC7y7oja46UgIbI4f8yVxbPrd1rJON3HcvOyUKJivIDXjjGZ1tksQ+PKHklZOJZEs+voUnDi0sRj/2dSb6vI3gRSnboBnIFavsqkUbDTdZ8rJxFWua5jsMzibppnO30aS5EB7xyTf/4n1L3tjtOSlJVyi2HIw+/T1WJg1ojx2Hf8KvTfKJ6sXZV6YpOa2S8OCEnhjfNy+m3/9v1V5D5XFTpM7zczbhlZ8PNny7nlDyIS35OPdc3Njw+vFZR0JcC9x3Xg9T9+EmZaOXyWc552XpcNulCWa5uDzn05AV/urdx2yWxFyMHlh2ig+VaY6bFJXTSDcxZUUo9FwrPYPVkdCl5InoCiJaR0T1RFQctOwBItpCRBuJ6Hx9YkYmlCVvBDeOKMSsu88yboOM7zCkEIdPegFGZ5d8YHxP08ISzcAsQ0uvu2YtgEsBvKZsJKLeAK4G0AdAewBziKi7EMKUjGHKk2Pk2/CaoR0NybHNMHosPL9Y8qEyUurBqWMs4TCrM63LkhdCrBdCbAyxaBKA94UQp4QQ2wFsATBUz74iobTkjZz63jLRmdEWjHuQE+alJcXH/BDPXOuO9Bx6cXoNCDVkpcaek9+sHEVm+eTzASjzhe6W2ppBRJOJaDkRLS8ri222plm3Roc22gogMEwwj03qi2nXDUZfqcC8n1EWsA+FWe4KK102E3TUljYrhDKqkieiOUS0NsTfJCMEEEJME0IUCyGKc3Jiq6oinB3irZkhheZOQmKsIyUxDuf1Ma6ovJsZF+U8VNWYU/8hwcJIOz2TFM1KjR7VJy+EGBvDdvcAUBZmLJDaTMGsUWm7sGOgrV16suX7ZPxFNP1nVhrh5AR3uF3NSkRnlrvmMwBXE1ESEXUG0A1AbElVVOABV15TbBhoG16Uhf/8xr+ZMP3MvPtGYdlDY0zfj10T7uTKX3JFMrOZ/39nY+mD5p9PteiKriGiSwC8CCAHQAkRrRRCnC+EWEdEHwD4EUAtgNvMiqwB3F/g2ikMKTQ+nzrjfIpMTLGsxO5J6CkWBVJ0zk61ZD9q0Rtd87EQokAIkSSEyBVCnK9Y9oQQoosQoocQYqZ+UcPjOUte5fHM/tHbmTbNpFtbaxSbkq6SMo02AOlZ7IoFlYxAv4SiBuOJGa9e88mrZdWucrtFcC2f3HaG5fsc0TUb8+4bhV+fVWT5vp2AXZa8bAT6ZVJZMN5Q8h7T8WpfWh+t2O37+p+xEmuWRL0U5aT516K0+cB9e97tFsAIPKfkVR7PvmNVKH58jrnCMIbjW4vSNm+NxxSERjyh5Ot9fhGVdM+13tfsNn5hd+EIf+p4wwvdqEXWDj497d5Q8l5T8VqPR2mp2N0ldgMpifbe9n69QnYdt/x42PWSsRtPKPl6j4XXXFlcoGn933602rB9j+oe26xjRj1+SnrXv6AxnYNdBsiwokA007m9c23Zv914omiI17hqSEdcWdwBNXUC3R+OHn2qzJ2v90Ga/gvT8sg5Brt94nnpKbbu30o+u30kfvH2d5i34SDsqsjZp306SqdOtGfnDsATSt6LPnkiQmK8OmWkPP4f9x03SySGiQn5/mRXoj14wl1jho6fMr6n8RuNwqMX9kaf9q01/+7sHm117bdHrjPcByO7Zpuy3fyMppazrGuSE1rgrjHdTNlnNE4vysJ953a3Zd9W40EbTBXJCS1QkGl/r80TSt5oS7506kRbKsrceEZnlNx5pubf6Z1GPeseZ1S/+tfNw/Dohb0N3+7Ht40I2b7hsfG4xyZF+97k4bjDpheM1a4L+en0myW/4bHxthkRSjzhrvGpodDA9G9LEUeEbhw+GZJgH7y/VI39CHbX2Io3lLxf+4MSG/ZX4P6PVqN1svsv56BOxufSD9Ytlw/uEHpFH9ArT7s7UC/3nNsdG/dXYGDHDMv3zXjEXeNzHd/A8apa1ev+9eoB5gmig9bJgbSwnbKMq8oVbD92NHDbbmPmXdrdgXoZ1DETyx4a25Dy1084ITbfE0r+VK3HSkNZgBNuvkgY+eJ2+rEyjJl4QsnzM6ydMT31ReSYhRnXUunOm9gvz/gduJDJPs2EaTXj+tpf+tETSr6eDXnN2JWFUS1mpY9+6dpBpmzXak4vytL1+wcn9DJIEiYSaUnxto+VeULJ17FT3jPIERhx3D1jPEILm0tieUPJG5i75oL+3J23k4LMFPxmVBe8eeMQw7bJPvnQfHzrCPx2nPWT/vyGMnT0Thvi5j2h5I2cDFWY5az6jH6DiDBlfE/L6o76mYEdM3HL2dZP+rOCjJbOieRRKvnzbEiS5gklb6QlzziHpHhjbk+7C0ibgV9LXqrFSZfc7o6krqeIiJ4hog1EtJqIPiaiDMWyB4hoCxFtJKLzI2xGN2pTDbdJTTRTDMeR0TIB94/rgeUPj7VblJh45+Zhhmwno2Ui3vvVcLx+fbEh27OCnj5KRxwLcS56c9stql5TaTaAvkKI/gA2AXgAAIioN4CrAfQBMA7Ay0QUp3NfYVE78BqcLMjrD1Kb1ETcenZXZKcl2S1KTBQXtjFsW6d3ycJYF+UT/9WZHOIYiamX9ou43En9HLvTOehS8kKIL4UQ8jTLJQDkaheTALwvhDglhNgOYAsA0xKVq3XXBL8L2qUnN1/HUbeHPtr7KG+517Ar97qZGBkx1So5ss/dSXa+Usnboe+NvJV+AUCucJEPYJdi2W6prRlENJmIlhPR8rKysph2rHbgNViBP3/VAM+mex3ds22TmPBVj5zXbJ27xnTD81cNsFAq7Vx4WvuGz0MKG/PaPGBDKmgz+OLuM/Hur5q7paIVNtEaa/D69cX49+Th2n5kMCmJxnXmz+/jnl6Z3S/sqLsnojlEtDbE3yTFOg8BqAXwjlYBhBDThBDFQojinJzYSs9V1WifDdU+PRkZLRNxRbE3k1W9eeMQpCtyhaSnJDRTJvec2x0XDwz57nUM7TMae1tK661ta3e6oILp2a41RnTJblby0WiLr2deKwzTOYHKCE7rkGHIdtwUFmu3uybqVCwhRMRROyK6EcAFAMaIxvnjewAotWeB1GYKmw5UqFqvc3Ya1u7xceUkF3qiTile4MqBc7tL+BlNsGUeTYl9v+OoidK4nw5tnJOELrdVMnYcrgQAlFfWWL5/vdE14wDcD+AiIUSlYtFnAK4moiQi6gygG4BlevYViUkD1Fmj1wxtfO/Iz5SLDALVjO0VOi+NfMzBlZKcTEJc4wWSrcC2rZIiXrdrhnZ0TLUrtQQPK4WLyFhw/zn4/I6RqFUxDnVVcQe0bRXo8ThtUnjffHNTHt891v5iHTLnSa6lG07vhN1HK5stf+zivqbuX6+36G8AWgGYTUQriehVABBCrAPwAYAfAXwB4DYhRJ3OfYVFbWUko+Ku7aBQQ3rc5ITIvk8j0/iajdJFIyu+xCjXcXzfdq57eQfXRAjXU+nQpiX65qer2uZTl/dHQpwz7/k/TjJXscXZ7QhXILtriCjkde1ocq9DV+YcIUTXCMueAPCEnu1bQfBgUFG2M2daFuWkofRwcysgFF4KDVXGQ6dJSdUGdsyM6Od0Y2H3YIlDWfJa4q1bSvd1cWEm9qw82fCdsZ7C7IAS79I2LeS9GW9yIL2zUxGaiHyuWycnYOZdZ6IwKxUb9h/HAIMGhozi0kH5+O+KPRjftx3mbTio6je3nB363etC3ddEyWe0TMSMO89EUU4q5qw/EPY3LjxMVS8mNYONf7ioD0Z1z2ko0PHUZf0x+awiZDl4rsSU8T0xdeYG3dt5/OK+mLVuPxZsPuSoanGje+bi41tHYECHDLy7bGdD+80jO+OGEYXYf7zK1P07p09jMcpwyl55rZGSGIeBHTNdNWofjnCzAeVjdtMhBls5vdu3RnJCXOSBV+Gu6AtA3QtYjcF3w4hCFGanNgxSJyfEoU97de4duzDKkv358E6Ove6yblHet6N7tUWHNi1NnxHrWyU/2qFFM4Ip7hSY9dmlrX43kqxI3BSZ0rt94wCdcsC4ui78EE9+ZgrODTP47FSUcwCA0C9ir4b7mhFi6Bw7vilKhS4PimelmtvL8q2SN3vgxyiuGdoBC+4/B4M66i9w7caIotOLsvDJbWfg8ztGoqviRVdTF/4x7p7bCr90WVqAnw/vFHH5pQPz8eQljVP5tQzEOx2teWgi5WJy+q2tfKHlZwSuoTznQxlJZiS+9ck7NeogGCIyLObXSX5KtRBRyHGSaF18NyWwAqK7ly4c0L7Jdy8lXtV6qdyaiymY4Bn4Zt2z7tB0jCF0kXK0X6xyXoGTOXYy8qSSRJe8xMPxXWnTyU7Bj7+X0ms71Y9uBspDtcrQdPeT4DNuOL1pl37No+fh7ZsCFZR65UWfXNKhTUuseuQ8XDa4IOq6TuenU7URlyfGt8Cq35+HTY+Pt0gi/Wx4bFzD573lJ5ssC1aEbuyVKVEejY90fBNkJW/2WJlv3TVuJLhWZKvkBLRMDFzCtCR1cdDKfDZupk5FuqJ0B1UHUoNyElvwYGTw419RFfkl5yb8VM83VK/F7LEytuRdRDTjLT8jxXFx/mYxf6O6OQNupUfQhLaTNU2jibykF01J4OXQjk5qiElpcq/MrEvqaUu+e24aNh040aStdOpEm6Qxn0VTRtstgmWUVZwK2f6rMztbLIk5tGvdtNZB66D86fFSV799ejL2HjN3Mo3ZRNPxE/q1w4w1+w3Zlt3ER/DDmzU2wZa8y3G7b9ZovDKIF5yALDgSQx54jTMp7M4sKkIMmEe7Zm6a1xELZj/BnlTyv7+gNwDgkQv7OCobnZl4/UEIJlyyuUlBoYZupSinadK92T82TeMgRxfdcY677m953EGZxiFa5GAs7203VXhrHHg1B8+5a2R3zC9GBrrtZ3TNxtebyvDDznIbpTIfN93URpCdloRth35q0rb0wTHIbd28pKMbaZXc9NE8ETTQmtEyAeWVNTi3dy5Kh7jHBRkv9TziFVkilfHhY3vlNstLpKZ3Jj/3Tjd15J53yHTgPPBqBE6/BSITyTUTLf0u4y6ieeHq692XhygcXnGxxYzJ9plnNMNjF/fF1UNC5/Z4aEIvFOWkoleed1LwyvTJT0dxp0zcd14Pu0WxlCcuaZ6Wwku6omvbNHRTpHEI1gMNXXwPHLTyCO4e261ZvQP3H2Fk0pLj0S8/Hc9dOcCU7XtGyV83vBOmXtY/5LLiwjaYd9/ZDTHlbiXUCz8tKR4f3jLCkNw2bqJbbqtmkVJeGpdITojD7HtHhV0u3wsuy94QEuV7qm9+Or7+f+c0fM/PSPHUy7uRxoOKa0H43x0jcW5vc4qTe0bJ+wEOpPEvwddeHri0u0i0EUR7Obv/CO2FlbyLuHNMN5wnve2v8EBqAiOQ62NeOigfWYpC327lrRuH4NejGjNoyoOSlw5qmm/oXzcPwzVDO3ii4lO099Rvx/fUvM16FTOi/YK7/Rc+I6dVEqZdX2y3GI7iuuGdcF2UNL1u4pyebXGOotZB77zWWLPnWEPpQ5lBHTM946KLZqnnpaegU1ZL7FBR/lLu2XCntxG25BnGwcjzATzglQlLpGOTezKhVgk1HiFvy6mTBO2QSpeSJ6LHiGg1Ea0koi+JqL3UTkT0AhFtkZYPMkZchvEXL1wzELec3QV9HV7CTw+napv7VoYWBiqi/XZcwFUTKopoWOesEFtzhyVv5UtbryX/jBCivxBiAIDPAfxeah8PoJv0NxnAKzr3wzC+pH1GCn47rmezDKReQja6leUdE+IDxxspa2qoIhuNlrxx8rkdXUpeCHFc8TUVjS/QSQD+IQIsAZBBRHl69sUwjLeJ5GIJ9YoL9QKQE7mFS3vhR3QPvBLREwCuB3AMgBzgmg9gl2K13VLbvhC/n4yAtY+OHTvqFYdhGJeQnpKAYydrkCoNKkcyvkO5N/4wqQ9K1uxrUgrykYt6o3tuGkZ1zzFYWmOwo4cR9XVHRHOIaG2Iv0kAIIR4SAjRAcA7AG7XKoAQYpoQolgIUZyT48wLwzCM8SjdM0BoBSjnZArlkw9Ovyy3/XpUF8e7t6yULqolL4QIXxq9Ke8AmAHgEQB7AChzDBRIbQzD+Ji89GSs3n0MKYlxzazzSEn2yiurm7XJerxjljGF7q1ATjzXtrV1xch1uWuIqJsQYrP0dRKADdLnzwDcTkTvAxgG4JgQopmrhmEYf/HMFadhYv8ydM9t1VzJR3BlHDrRXMnHx7XA368vxmkd3BN5NKJLFp678jRM6GfdEKVen/xUIuoBoB7ADgC/kdpnAJgAYAuASgA36dwPwzAeoHVyAi46LZDzPzglg1LHq81DZFa+F7MgIlw6yNrZ6rqUvBDisjDtAsBterbNMIy3CVbjSkveb/URzITjjBiGcQis2M2AlTzDMLYgR8zI8fFKS95LaaPthpU8wzC2MLhTIMFafmYglHJYUZtm6/DMVf1wFkqGYWxhyvieuKK4AD3btcace89CQWbzUEgvJ2azClbyDMPYQkJcC/Rs1xoA0LWt90pzOgV21zAMw3gYVvIMwzieeIenKXAyrOQZhnE87JuPHVbyDMM4jon9A9P+O2enAgBGds22UxxXw0qeYRjHcfWQDtj4+LiGiJvTu4SqAsWogZU8wzCOg4iQFB/X8J3j5WOHlTzDMIyHYSXPMAzjYVjJMwzDeBhW8gzDOB52yccOK3mGYRgPw0qeYRjHw9E1scNKnmEYx5PRMgEA0K51ss2SuA9W8gzDOJ44Ka/Bmd145qtWWMkzDON4uOZr7Bii5InoPiISRJQtfScieoGIthDRaiIaZMR+GIZhGG3oVvJE1AHAeQB2KprHA+gm/U0G8Ire/TAMwzDaMcKS/wuA+9E0lHUSgH+IAEsAZBBRngH7YhjGh8S3CKiqxHj2MGtFV/k/IpoEYI8QYhU1TficD2CX4vtuqW1fiG1MRsDaR8eOHfWIwzCMR7loQHtsOliB287parcoriOqkieiOQDahVj0EIAHEXDVxIwQYhqAaQBQXFzMoysMwzQjIa4FHhjfy24xXElUJS+EGBuqnYj6AegMQLbiCwCsIKKhAPYA6KBYvUBqYxiGYSwkZgeXEGKNEKKtEKJQCFGIgEtmkBBiP4DPAFwvRdkMB3BMCNHMVcMwDMOYiy6ffARmAJgAYAuASgA3mbQfhmEYJgKGKXnJmpc/CwC3GbVthmEYJjY4HolhGMbDsJJnGIbxMKzkGYZhPAwreYZhGA9DwkHZ+ImoDMCOGH+eDeCQgeJ4BT4vzeFzEho+L81xyznpJITICbXAUUpeD0S0XAhRbLccToPPS3P4nISGz0tzvHBO2F3DMAzjYVjJMwzDeBgvKflpdgvgUPi8NIfPSWj4vDTH9efEMz55hmEYpjlesuQZhmGYIFjJMwzDeBhPKHkiGkdEG6XC4VPslsdoiOhNIjpIRGsVbW2IaDYRbZb+Z0rtYYuoE9EN0vqbiegGRftgIloj/eYFCirz5USIqAMRzSeiH4loHRHdJbX7/bwkE9EyIlolnZc/SO2diWipdCz/JqJEqT1J+r5FWl6o2NYDUvtGIjpf0e7K542I4ojoByL6XPruj3MihHD1H4A4AFsBFAFIBLAKQG+75TL4GM8CMAjAWkXb0wCmSJ+nAHhK+jwBwEwABGA4gKVSexsA26T/mdLnTGnZMmldkn473u5jVnFO8hCoXwAArQBsAtCbzwsIQJr0OQHAUukYPgBwtdT+KoBbpM+3AnhV+nw1gH9Ln3tLz1ISAsWBtkrPmmufNwD3AngXwOfSd1+cEy9Y8kMBbBFCbBNCVAN4H4FC4p5BCPENgCNBzZMATJc+TwdwsaI9VBH18wHMFkIcEUIcBTAbwDhpWWshxBIRuJP/odiWYxFC7BNCrJA+VwBYj0AdYb+fFyGEOCF9TZD+BIDRAD6U2oPPi3y+PgQwRuqxTALwvhDilBBiOwK1IYbCpc8bERUAmAjgdek7wSfnxAtKPlzRcK+TKxqrbe0HkCt9Dnc+IrXvDtHuGqTu9EAErFbfnxfJLbESwEEEXlpbAZQLIWqlVZTH0nD80vJjALKg/Xw5necB3A+gXvqeBZ+cEy8oed8jWZq+jIUlojQAHwG4WwhxXLnMr+dFCFEnhBiAQG3loQB62iuRvRDRBQAOCiG+t1sWO/CCkvdr0fADkksB0v+DUnu48xGpvSBEu+MhogQEFPw7Qoj/Ss2+Py8yQohyAPMBnI6Ae0quBKc8lobjl5anAzgM7efLyZwB4CIiKkXAlTIawF/hl3Ni96CA3j8EShhuQ2AgRB706GO3XCYcZyGaDrw+g6YDjE9Lnyei6QDjMqm9DYDtCAwuZkqf20jLggcYJ9h9vCrOByHgJ38+qN3v5yUHQIb0OQXAAgAXAPgPmg4y3ip9vg1NBxk/kD73QdNBxm0IDDC6+nkDcDYaB159cU5sF8CgCzcBgeiKrQAeslseE47vPQD7ANQg4O/7JQI+wrkANgOYo1BMBOAl6VysAVCs2M4vEBgs2gLgJkV7MYC10m/+BmkmtJP/AIxEwBWzGsBK6W8Cnxf0B/CDdF7WAvi91F6EwEtri6TckqT2ZOn7Fml5kWJbD0nHvhGKyCI3P29BSt4X54TTGjAMw3gYL/jkGYZhmDCwkmcYhvEwrOQZhmE8DCt5hmEYD8NKnmEYxsOwkmeYIIhIENGfFd//j4getVEkhokZVvIM05xTAC4lomy7BWEYvbCSZ5jm1CJQ2/Oe4AVEVEhE86Sc9HOJqKP14jGMeljJM0xoXgJwLRGlB7W/CGC6EKI/gHcAvGC5ZAyjAZ7xyjBBENEJIUQaEf0RgVQSJxEoxPEoER0CkCeEqJESpO0TQrBbh3EsbMkzTHieRyBPUKrNcjBMzLCSZ5gwCCGOIFAi7peK5m8RyEwIANcikOWRYRwLK3mGicyfASjdMXcAuImIVgO4DsBdtkjFMCphnzzDMIyHYUueYRjGw7CSZxiG8TCs5BmGYTwMK3mGYRgPw0qeYRjGw7CSZxiG8TCs5BmGYTzM/wf+6CeqtMeJ8gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.DEWP.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['pm2.5'] = df['pm2.5']/1000\n",
    "df['PRES'] = df['PRES']/1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsample_sequence(df, length):\n",
    "    value = df.sample(1).index[0]\n",
    "    idx_position = df.index.get_loc(value)\n",
    "    if len(df) - length < idx_position:\n",
    "        return subsample_sequence(df, length)    \n",
    "    return df.loc[value:value+length-1]\n",
    "\n",
    "df_subsample = subsample_sequence(df, length=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_subsample_sequence(df, length):\n",
    "    df_subsample = subsample_sequence(df, length)\n",
    "    X_subsample = df_subsample[0:-1]\n",
    "    y_subsample = df_subsample[-1:]['pm2.5'].iloc[0]\n",
    "    if y_subsample == np.nan:\n",
    "        return split_subsample_sequence(df, length)\n",
    "    if X_subsample.isna()['pm2.5'].sum() == 0:\n",
    "        pass\n",
    "    elif X_subsample.isna()['pm2.5'].sum() == length:\n",
    "        X_subsample.fillna(df['pm2.5'].mean(), inplace=True)\n",
    "    else:\n",
    "        X_subsample.fillna(X_subsample['pm2.5'].mean(), inplace=True)\n",
    "    return X_subsample.to_numpy(), y_subsample\n",
    "\n",
    "(X_sample, y_sample) = split_subsample_sequence(df, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_X_y(df, number_of_sequences, length):\n",
    "    for i in range(number_of_sequences):\n",
    "        df_subsample = split_subsample_sequence(df, length)\n",
    "        if i == 0:\n",
    "            X =np.vstack(([df_subsample[0]],))\n",
    "            y =np.vstack(([df_subsample[1]],))\n",
    "        else:\n",
    "            X =np.vstack(([df_subsample[0]],X))\n",
    "            y =np.vstack(([df_subsample[1]],y))\n",
    "    return X, y[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = get_X_y(df, 100, 21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 20, 7)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = get_X_y(df, 2000, 21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8765, 7)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(35059, 7)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df.iloc[0:round(len(df) * 0.8)]\n",
    "df_test = df.iloc[round(len(df) * 0.8):]\n",
    "print(df_test.shape)\n",
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = get_X_y(df_train, 2000, 100)\n",
    "X_test, y_test = get_X_y(df_test, 400, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model():\n",
    "    metric = metrics.MAPE\n",
    "\n",
    "    model = Sequential()\n",
    "\n",
    "    normalizer = Normalization()\n",
    "    normalizer.adapt(X_train)\n",
    "    \n",
    "    model.add(normalizer)\n",
    "    model.add(LSTM(units=10, activation='tanh'))\n",
    "    model.add(layers.Dense(20, activation=\"relu\"))\n",
    "    model.add(layers.Dense(1, activation=\"linear\"))\n",
    "\n",
    "    model.compile(loss='mse',\n",
    "                  metrics=metric,\n",
    "                  optimizer='rmsprop') # Use `rmsprop`\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "100/100 [==============================] - 2s 14ms/step - loss: 0.0064 - mean_absolute_percentage_error: 134.9958 - val_loss: 0.0020 - val_mean_absolute_percentage_error: 83.9409\n",
      "Epoch 2/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 0.0020 - mean_absolute_percentage_error: 72.4182 - val_loss: 0.0014 - val_mean_absolute_percentage_error: 64.2567\n",
      "Epoch 3/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 0.0015 - mean_absolute_percentage_error: 56.5905 - val_loss: 0.0012 - val_mean_absolute_percentage_error: 55.3846\n",
      "Epoch 4/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 0.0013 - mean_absolute_percentage_error: 50.8397 - val_loss: 0.0011 - val_mean_absolute_percentage_error: 51.3823\n",
      "Epoch 5/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 0.0011 - mean_absolute_percentage_error: 48.1010 - val_loss: 9.4290e-04 - val_mean_absolute_percentage_error: 43.5520\n",
      "Epoch 6/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 0.0010 - mean_absolute_percentage_error: 44.5478 - val_loss: 8.7767e-04 - val_mean_absolute_percentage_error: 44.3578\n",
      "Epoch 7/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 9.3320e-04 - mean_absolute_percentage_error: 42.1382 - val_loss: 9.2871e-04 - val_mean_absolute_percentage_error: 51.4571\n",
      "Epoch 8/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 8.6339e-04 - mean_absolute_percentage_error: 40.0440 - val_loss: 7.9108e-04 - val_mean_absolute_percentage_error: 38.6106\n",
      "Epoch 9/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 8.1449e-04 - mean_absolute_percentage_error: 38.0737 - val_loss: 9.0915e-04 - val_mean_absolute_percentage_error: 46.6879\n",
      "Epoch 10/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 7.7893e-04 - mean_absolute_percentage_error: 37.0187 - val_loss: 7.8631e-04 - val_mean_absolute_percentage_error: 43.9923\n",
      "Epoch 11/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 7.5026e-04 - mean_absolute_percentage_error: 36.7164 - val_loss: 7.7792e-04 - val_mean_absolute_percentage_error: 35.6648\n",
      "Epoch 12/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 7.3656e-04 - mean_absolute_percentage_error: 36.0206 - val_loss: 8.0179e-04 - val_mean_absolute_percentage_error: 52.3452\n",
      "Epoch 13/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 7.0753e-04 - mean_absolute_percentage_error: 34.7360 - val_loss: 7.0518e-04 - val_mean_absolute_percentage_error: 35.1900\n",
      "Epoch 14/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.8272e-04 - mean_absolute_percentage_error: 34.2662 - val_loss: 7.3943e-04 - val_mean_absolute_percentage_error: 44.5625\n",
      "Epoch 15/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.7438e-04 - mean_absolute_percentage_error: 34.0648 - val_loss: 7.4398e-04 - val_mean_absolute_percentage_error: 45.3164\n",
      "Epoch 16/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 6.6956e-04 - mean_absolute_percentage_error: 32.6215 - val_loss: 6.3193e-04 - val_mean_absolute_percentage_error: 32.6762\n",
      "Epoch 17/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.6114e-04 - mean_absolute_percentage_error: 33.2050 - val_loss: 6.2267e-04 - val_mean_absolute_percentage_error: 32.2302\n",
      "Epoch 18/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.3292e-04 - mean_absolute_percentage_error: 32.4874 - val_loss: 8.6690e-04 - val_mean_absolute_percentage_error: 39.9021\n",
      "Epoch 19/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.3133e-04 - mean_absolute_percentage_error: 31.9268 - val_loss: 7.7817e-04 - val_mean_absolute_percentage_error: 35.3758\n",
      "Epoch 20/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 6.2153e-04 - mean_absolute_percentage_error: 31.1629 - val_loss: 7.8785e-04 - val_mean_absolute_percentage_error: 45.4716\n",
      "Epoch 21/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 6.1294e-04 - mean_absolute_percentage_error: 30.5037 - val_loss: 6.8053e-04 - val_mean_absolute_percentage_error: 33.7377\n",
      "Epoch 22/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.9786e-04 - mean_absolute_percentage_error: 30.7037 - val_loss: 6.5622e-04 - val_mean_absolute_percentage_error: 32.6355\n",
      "Epoch 23/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.9545e-04 - mean_absolute_percentage_error: 30.8988 - val_loss: 6.4028e-04 - val_mean_absolute_percentage_error: 34.6455\n",
      "Epoch 24/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.9548e-04 - mean_absolute_percentage_error: 30.6254 - val_loss: 6.8384e-04 - val_mean_absolute_percentage_error: 28.8588\n",
      "Epoch 25/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.8120e-04 - mean_absolute_percentage_error: 30.5436 - val_loss: 9.4191e-04 - val_mean_absolute_percentage_error: 34.4393\n",
      "Epoch 26/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.7660e-04 - mean_absolute_percentage_error: 30.4751 - val_loss: 6.0936e-04 - val_mean_absolute_percentage_error: 32.0873\n",
      "Epoch 27/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.6144e-04 - mean_absolute_percentage_error: 30.2823 - val_loss: 6.2201e-04 - val_mean_absolute_percentage_error: 31.9639\n",
      "Epoch 28/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.6784e-04 - mean_absolute_percentage_error: 30.0142 - val_loss: 6.4898e-04 - val_mean_absolute_percentage_error: 30.0324\n",
      "Epoch 29/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.4810e-04 - mean_absolute_percentage_error: 28.9931 - val_loss: 6.0687e-04 - val_mean_absolute_percentage_error: 30.7064\n",
      "Epoch 30/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.4840e-04 - mean_absolute_percentage_error: 29.6667 - val_loss: 6.4003e-04 - val_mean_absolute_percentage_error: 33.4189\n",
      "Epoch 31/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.6262e-04 - mean_absolute_percentage_error: 29.7608 - val_loss: 6.6769e-04 - val_mean_absolute_percentage_error: 31.4280\n",
      "Epoch 32/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.5324e-04 - mean_absolute_percentage_error: 29.0013 - val_loss: 6.2148e-04 - val_mean_absolute_percentage_error: 32.4301\n",
      "Epoch 33/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.2099e-04 - mean_absolute_percentage_error: 29.0593 - val_loss: 6.0115e-04 - val_mean_absolute_percentage_error: 29.5120\n",
      "Epoch 34/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.3065e-04 - mean_absolute_percentage_error: 28.7431 - val_loss: 6.5594e-04 - val_mean_absolute_percentage_error: 30.5357\n",
      "Epoch 35/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.3500e-04 - mean_absolute_percentage_error: 29.2231 - val_loss: 6.0660e-04 - val_mean_absolute_percentage_error: 36.1572\n",
      "Epoch 36/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.3581e-04 - mean_absolute_percentage_error: 28.6999 - val_loss: 5.9628e-04 - val_mean_absolute_percentage_error: 27.0751\n",
      "Epoch 37/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.3058e-04 - mean_absolute_percentage_error: 27.7157 - val_loss: 6.1183e-04 - val_mean_absolute_percentage_error: 28.6516\n",
      "Epoch 38/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.2251e-04 - mean_absolute_percentage_error: 28.5641 - val_loss: 6.0070e-04 - val_mean_absolute_percentage_error: 29.7883\n",
      "Epoch 39/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.1453e-04 - mean_absolute_percentage_error: 28.3066 - val_loss: 6.0163e-04 - val_mean_absolute_percentage_error: 30.3801\n",
      "Epoch 40/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.0787e-04 - mean_absolute_percentage_error: 28.4253 - val_loss: 5.7687e-04 - val_mean_absolute_percentage_error: 30.4448\n",
      "Epoch 41/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.1980e-04 - mean_absolute_percentage_error: 29.0237 - val_loss: 6.1015e-04 - val_mean_absolute_percentage_error: 28.7140\n",
      "Epoch 42/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.0171e-04 - mean_absolute_percentage_error: 28.9965 - val_loss: 6.0396e-04 - val_mean_absolute_percentage_error: 26.1296\n",
      "Epoch 43/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.0322e-04 - mean_absolute_percentage_error: 28.3446 - val_loss: 6.2044e-04 - val_mean_absolute_percentage_error: 27.4375\n",
      "Epoch 44/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.9278e-04 - mean_absolute_percentage_error: 27.6396 - val_loss: 5.9066e-04 - val_mean_absolute_percentage_error: 32.6155\n",
      "Epoch 45/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 5.0315e-04 - mean_absolute_percentage_error: 28.1622 - val_loss: 5.7665e-04 - val_mean_absolute_percentage_error: 29.1544\n",
      "Epoch 46/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.8199e-04 - mean_absolute_percentage_error: 26.9987 - val_loss: 5.9202e-04 - val_mean_absolute_percentage_error: 27.3992\n",
      "Epoch 47/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 4.7717e-04 - mean_absolute_percentage_error: 27.5117 - val_loss: 5.8477e-04 - val_mean_absolute_percentage_error: 36.0539\n",
      "Epoch 48/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 5.0035e-04 - mean_absolute_percentage_error: 28.3700 - val_loss: 7.4463e-04 - val_mean_absolute_percentage_error: 31.6310\n",
      "Epoch 49/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.8483e-04 - mean_absolute_percentage_error: 27.7824 - val_loss: 6.6400e-04 - val_mean_absolute_percentage_error: 29.5597\n",
      "Epoch 50/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.7428e-04 - mean_absolute_percentage_error: 27.6882 - val_loss: 6.2853e-04 - val_mean_absolute_percentage_error: 29.3314\n",
      "Epoch 51/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.7963e-04 - mean_absolute_percentage_error: 27.2849 - val_loss: 7.5928e-04 - val_mean_absolute_percentage_error: 43.8197\n",
      "Epoch 52/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.7648e-04 - mean_absolute_percentage_error: 27.7039 - val_loss: 5.7852e-04 - val_mean_absolute_percentage_error: 27.0551\n",
      "Epoch 53/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.7046e-04 - mean_absolute_percentage_error: 27.1398 - val_loss: 5.6906e-04 - val_mean_absolute_percentage_error: 28.8734\n",
      "Epoch 54/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.7376e-04 - mean_absolute_percentage_error: 27.0591 - val_loss: 6.8208e-04 - val_mean_absolute_percentage_error: 30.7832\n",
      "Epoch 55/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.6495e-04 - mean_absolute_percentage_error: 28.1537 - val_loss: 5.9996e-04 - val_mean_absolute_percentage_error: 29.8573\n",
      "Epoch 56/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.5569e-04 - mean_absolute_percentage_error: 26.8070 - val_loss: 6.7713e-04 - val_mean_absolute_percentage_error: 32.8005\n",
      "Epoch 57/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.6198e-04 - mean_absolute_percentage_error: 27.0434 - val_loss: 6.1002e-04 - val_mean_absolute_percentage_error: 29.9545\n",
      "Epoch 58/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.5325e-04 - mean_absolute_percentage_error: 26.9099 - val_loss: 5.8258e-04 - val_mean_absolute_percentage_error: 27.2637\n",
      "Epoch 59/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 4.6141e-04 - mean_absolute_percentage_error: 27.4420 - val_loss: 5.9429e-04 - val_mean_absolute_percentage_error: 28.4101\n",
      "Epoch 60/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.5195e-04 - mean_absolute_percentage_error: 27.5584 - val_loss: 5.8466e-04 - val_mean_absolute_percentage_error: 31.5207\n",
      "Epoch 61/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 4.6018e-04 - mean_absolute_percentage_error: 26.1472 - val_loss: 6.8762e-04 - val_mean_absolute_percentage_error: 28.9811\n",
      "Epoch 62/1000\n",
      "100/100 [==============================] - 1s 12ms/step - loss: 4.5141e-04 - mean_absolute_percentage_error: 27.0325 - val_loss: 5.8950e-04 - val_mean_absolute_percentage_error: 27.6594\n",
      "Epoch 63/1000\n",
      "100/100 [==============================] - 1s 11ms/step - loss: 4.4012e-04 - mean_absolute_percentage_error: 25.8969 - val_loss: 7.0121e-04 - val_mean_absolute_percentage_error: 33.4274\n",
      "13/13 [==============================] - 0s 5ms/step - loss: 7.6031e-04 - mean_absolute_percentage_error: 34.6537\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0007603096892125905, 34.6536865234375]"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = init_model()\n",
    "\n",
    "es = EarlyStopping(patience=10, restore_best_weights=True)\n",
    "\n",
    "history = model.fit(X_train, y_train,\n",
    "              epochs=1000,  # Use early stop in practice\n",
    "              batch_size=16, \n",
    "              verbose=1,\n",
    "              callbacks=[es],\n",
    "              validation_split=0.2)\n",
    "\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benchmark MAPE on the test set : 240 %\n",
      "Benchmark MAPE on the test set : 19 %\n",
      "MAPE on the test set : 38 %\n"
     ]
    }
   ],
   "source": [
    "y_pred = np.mean(y_train)\n",
    "bench_res = np.mean(np.abs(y_pred - y_test)/y_test)*100\n",
    "\n",
    "print(f'Benchmark MAPE on the test set : {bench_res:.0f} %')\n",
    "\n",
    "y_pred_baseline = X_test[:,-1,0]\n",
    "bench_res = np.mean(np.abs(y_pred_baseline - y_test)/y_test)*100\n",
    "print(f'Benchmark MAPE on the test set : {bench_res:.0f} %')\n",
    "\n",
    "res = model.evaluate(X_test, y_test, verbose=0)\n",
    "\n",
    "print(f'MAPE on the test set : {res[1]:.0f} %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f5ef0362fa0>]"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnbElEQVR4nO3de3hV1Z3/8ff3nJPkQC5ckyByVxBBFCxeR61WRbTe63X6dKx1foxTHafttPOznd901Gk7tZ2pra3TGTvaWusFa2tLq462Yq2itQTkIiIXUbmTEELIPTk5398f+wRCSEiAhBP2/rye5zzZZ++Vk8V5Dp+9ztprrW3ujoiIhFcs2xUQEZG+paAXEQk5Bb2ISMgp6EVEQk5BLyIScolsV6Cj4cOH+7hx47JdDRGRI8qiRYu2u3txZ8f6XdCPGzeOsrKybFdDROSIYmYfdnVMXTciIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyCnoRkZBT0IuIhFxogn7zzga+8+IqPthel+2qiIj0K6EJ+qr6Zu6fv5Z3t+7KdlVERPqV0AR9aVESgG27mrJcExGR/iU0QT90YC6JmLFtV2O2qyIi0q+EJuhjMaOkME8tehGRDkIT9AAlRUnKa9SiFxFpL1RBX1qUp64bEZEOQhb0SXXdiIh0ELqgr25oobGlNdtVERHpN0IV9CWFeQCUq1UvIrJbqIJ+91h6XZAVEdktnEGvC7IiIruFLOiDrhtdkBUR2SNUQT9oQA65iRjlatGLiOwWqqA3M42lFxHpIFRBD1BaqLH0IiLthS7oS4ryNOpGRKSdHgW9mc02s1VmttbM7uzkeJ6Zzc0cf9PMxmX2f9LMlrR7pM1seu/+E/ZWUpikQi16EZHdug16M4sDDwAXA1OAG81sSoditwBV7n4scB9wL4C7P+bu0919OvAp4H13X9J71d9XaVGSmqYUdU2pvvwzIiJHjJ606E8F1rr7OndvBp4EruhQ5grgkcz208D5ZmYdytyY+d0+1TbEsrxGrXoREehZ0B8NbGj3fGNmX6dl3D0FVAPDOpS5Hniisz9gZnPMrMzMyioqKnpS7y5p0pSIyN4Oy8VYMzsNqHf3tzs77u4PuvtMd59ZXFx8SH9rz6QpBb2ICPQs6DcBo9s9H5XZ12kZM0sAg4DKdsdvoIvWfG8rybTotbCZiEigJ0G/EJhoZuPNLJcgtOd1KDMPuCmzfQ0w390dwMxiwHUchv55gMK8BANy4mrRi4hkJLor4O4pM7sdeAGIAw+7+wozuwcoc/d5wEPAo2a2FthBcDJocw6wwd3X9X7197V7dqwuxoqIAD0IegB3fw54rsO+r7bbbgSu7eJ3/wCcfvBVPHAlRUm16EVEMkI3MxaCkTda2ExEJBDOoC/MY9uuJjKXCUREIi2cQV+UpKGllRrNjhURCWfQl7TNjlX3jYhISIO+sG12rEbeiIiEMug1O1ZEZI9QBv3u2bEaSy8iEs6gL8hLUJCXUIteRISQBj0EF2S13o2ISIiDPrh3rFr0IiLhDXrdO1ZEBAh10Cc1O1ZEhBAHfUlRkuZUmuqGlmxXRUQkq0Ib9HvG0uuCrIhEW4iDXveOFRGBMAd9oYJeRARCHPS7FzbT7FgRibjQBn0yJ86gATlq0YtI5IU26CEzll5BLyIRF+qgLylMatSNiEReuIO+KE83HxGRyAt10JcWJamobSKd1uxYEYmucAd9YR4trU5VfXO2qyIikjXhDvoi3VJQRKRHQW9ms81slZmtNbM7OzmeZ2ZzM8ffNLNx7Y6daGZvmNkKM1tuZslerP9+td1pSqtYikiUdRv0ZhYHHgAuBqYAN5rZlA7FbgGq3P1Y4D7g3szvJoCfAbe6+1TgXOCwrTLWtt6NLsiKSJT1pEV/KrDW3de5ezPwJHBFhzJXAI9ktp8GzjczA2YBy9x9KYC7V7p7a+9UvXvFhVrYTESkJ0F/NLCh3fONmX2dlnH3FFANDAMmAW5mL5jZYjP7x87+gJnNMbMyMyurqKg40H9Dl/IScYbm52rSlIhEWl9fjE0AZwGfzPy8yszO71jI3R9095nuPrO4uLhXK1BSmKcWvYhEWk+CfhMwut3zUZl9nZbJ9MsPAioJWv9/dPft7l4PPAecfKiVPhClRUnKdTFWRCKsJ0G/EJhoZuPNLBe4AZjXocw84KbM9jXAfA/u4fcCMM3MBmZOAB8F3umdqveM1rsRkahLdFfA3VNmdjtBaMeBh919hZndA5S5+zzgIeBRM1sL7CA4GeDuVWb2HYKThQPPufuzffRv6VRpUZKKmiZa0048ZofzT4uI9AvdBj2Auz9H0O3Sft9X2203Atd28bs/IxhimRUlRUnSDpW1TbvH1YuIREmoZ8ZCsAwCaIiliERX6IO+RPeOFZGIC33Qt82O1TIIIhJVoQ/64QV5mEG5um5EJKJCH/Q58RjD8vM0ll5EIiv0QQ9tY+nVoheRaIpI0Cd1MVZEIisiQa8WvYhEVySCvqQwSWVdEy2t6WxXRUTksItE0JcWJXGH7bVq1YtI9EQk6DU7VkSiKyJBr9mxIhJdkQj6Et07VkQiLBJBPyw/j3jM1HUjIpEUiaCPx4ziAt2ARESiKRJBD0H3zbYatehFJHqiE/SFSfXRi0gkRSbode9YEYmqCAV9kqr6FppSrdmuiojIYRWhoA+GWFaon15EIiYyQb/nloIKehGJlsgEfWlhEPS6ICsiUROdoN+93o2CXkSiJTJBP2RgLjlx01h6EYmcHgW9mc02s1VmttbM7uzkeJ6Zzc0cf9PMxmX2jzOzBjNbknn8Vy/Xv8diMaOkUHeaEpHoSXRXwMziwAPAhcBGYKGZzXP3d9oVuwWocvdjzewG4F7g+syx99x9eu9W++CUFOVRrouxIhIxPWnRnwqsdfd17t4MPAlc0aHMFcAjme2ngfPNzHqvmr2jVC16EYmgngT90cCGds83ZvZ1WsbdU0A1MCxzbLyZvWVmr5jZ2Z39ATObY2ZlZlZWUVFxQP+AA6HZsSISRX19MXYLMMbdZwBfAB43s6KOhdz9QXef6e4zi4uL+6wyJUVJdjWmaGjW7FgRiY6eBP0mYHS756My+zotY2YJYBBQ6e5N7l4J4O6LgPeASYda6YPVdqep8hq16kUkOnoS9AuBiWY23sxygRuAeR3KzANuymxfA8x3dzez4szFXMxsAjARWNc7VT9wJYW6d6yIRE+3o27cPWVmtwMvAHHgYXdfYWb3AGXuPg94CHjUzNYCOwhOBgDnAPeYWQuQBm519x198Q/pCd07VkSiqNugB3D354DnOuz7arvtRuDaTn7vF8AvDrGOvUazY0UkiiIzMxZg0IAcchMxyjU7VkQiJFJBb2aUFuVpYTMRiZRIBT20TZpSi15EoiN6QV+UZJuGV4pIhEQu6LXejYhETeSCvrQoSW1TitqmVLarIiJyWEQw6IMhlrogKyJREb2gL9S9Y0UkWiIX9CVa70ZEIiZyQa/ZsSISNZEL+oK8BANz4+q6EZHIiFzQB7NjdacpEYmOyAU9QHGhxtKLSHREMug1O1ZEoiSaQV8Y3DvW3bNdFRGRPhfNoC9K0tiSZlejZseKSPhFMuhLMkMsK9R9IyIREMmg33NLQV2QFZHwi3jQq0UvIuEXyaAvKWybHasWvYiEXySDPj8vQWFeQi16EYmESAY9ZG5AoouxIhIBkQ36YBkEdd2ISPhFPOjVoheR8OtR0JvZbDNbZWZrzezOTo7nmdnczPE3zWxch+NjzKzWzL7YS/U+ZG33jtXsWBEJu26D3sziwAPAxcAU4EYzm9Kh2C1AlbsfC9wH3Nvh+HeA5w+9ur2ntDBJc2uanfUt2a6KiEif6kmL/lRgrbuvc/dm4Engig5lrgAeyWw/DZxvZgZgZlcC7wMreqXGXdn6Njw0CypW96h42+xYLW4mImHXk6A/GtjQ7vnGzL5Oy7h7CqgGhplZAfB/gbv39wfMbI6ZlZlZWUVFRU/rvrf84bBxISyb26Pimh0rIlHR1xdj7wLuc/fa/RVy9wfdfaa7zywuLj64v1Q4AiacB8uegnS62+J7bhKuFr2IhFtPgn4TMLrd81GZfZ2WMbMEMAioBE4DvmVmHwCfA75iZrcfWpX346QboHo9rH+j26JtXTflCnoRCbmeBP1CYKKZjTezXOAGYF6HMvOAmzLb1wDzPXC2u49z93HAd4FvuPsPeqfqnZj8ccjJh2VPdls0mRNn0IAcdd2ISOh1G/SZPvfbgReAlcBT7r7CzO4xs8szxR4i6JNfC3wB2GcI5mGRmw9TLocVv4aW7lvqpZodKyIRkOhJIXd/Dniuw76vtttuBK7t5jXuOoj6HbgTr4elT8Dq52HqVfstqtmxIhIF4ZsZO/4cKDwKlnY/+qakMKk+ehEJvfAFfSwO066Ftb+Duu37LRp03TSRTmt2rIiEV/iCHoLRN+kUvP3L/RYrLUqSSjs76psPU8VERA6/cAZ96VQondbt6JvSttmx6r4RkRALZ9ADnHQ9bFoE29d0WaQkMzu2XBdkRSTEwhv0064Fi+13SQTdO1ZEoiC8QV84AiacGwR9F0siFBfo3rEiEn7hDXqAE2+Anethw586PZybiDEsP1crWIpIqIU76I+/NFgSYWnXF2WLC/M0ll5EQi3cQZ+bD8dfBit+1eWSCJodKyJhF+6gh2D0TVN1sCRCJ0qL8nQxVkRCLfxBP/6j+10SobQoyfbaJlKt3a9hLyJyJAp/0MfiMO2aLpdEKClKknaorNPsWBEJp/AHPQSjb7pYEqG0sO0GJOqnF5FwikbQjzgBSk/odEkETZoSkbCLRtBDsE59J0si7A56jaUXkZCKTtB3sSTC8IJczDQ7VkTCKzpBX3RUMAKnw5IIiXiM4QWaNCUi4RWdoIdgnfpOlkTQWHoRCbNoBf3kSyFn4D5LIkwqKeSNdZWs2VaTpYqJiPSdaAV9XkGnSyLcefFk8nMT3Pb4YhqaW7NXPxGRPhCtoIdg9E1TNaz+3927SoqS3Hf9dNaU13LXvBVZrJyISO+LXtBPOBcKRuwz+uacScV89txjmFu2gV+9tSk7dRMR6QPRC/q2JRHWvAh1lXsd+vwFkzhl3BD+6ZnlrKuozVIFRUR6V4+C3sxmm9kqM1trZnd2cjzPzOZmjr9pZuMy+081syWZx1Izu6qX639wTsosibBi7yUREvEY9984g9xEjNsef4vGFvXXi8iRr9ugN7M48ABwMTAFuNHMpnQodgtQ5e7HAvcB92b2vw3MdPfpwGzgv80s0Ut1P3gjpkHJ1E5vSHLUoAH8x3UnsXLLLr727DtZqJyISO/qSYv+VGCtu69z92bgSeCKDmWuAB7JbD8NnG9m5u717p7K7E8C3huV7hUnXQ+bymD72n0OfWxyKXPOmcDP/rSeZ5dtyULlRER6T0+C/mhgQ7vnGzP7Oi2TCfZqYBiAmZ1mZiuA5cCt7YJ/NzObY2ZlZlZWUVFx4P+KgzHtWsD2uSjb5ksXHcf00YO58xfLWF9Zf3jqJCLSB/r8Yqy7v+nuU4FTgC+bWbKTMg+6+0x3n1lcXNzXVQoUjQxG4CybC77vF42ceIzv3zgDM7j9icU0pdRfLyJHpp4E/SZgdLvnozL7Oi2T6YMfBOw1pMXdVwK1wAkHW9led9INsPNDWP+nTg+PHjqQb197Ess2VnPv86sOc+VERHpHT4J+ITDRzMabWS5wAzCvQ5l5wE2Z7WuA+e7umd9JAJjZWGAy8EGv1Lw3tC2J0Mk69W0umjqCT585jocXvM+LK7YexsqJiPSOboM+06d+O/ACsBJ4yt1XmNk9ZnZ5pthDwDAzWwt8AWgbgnkWsNTMlgDPAJ91933v55ctu5dEeGavJRE6+vIlk5l29CC++POlbKxSf72IHFnMO+mfzqaZM2d6WVnZ4fuDa1+Cn10N1/0UpnQcTLTHh5V1fPz+15hYWsBTf3MGOfHozTUTkf7LzBa5+8zOjimt2pZEeO27UNN118zYYfl88xPTeGv9Tv79RfXXi8iRQ0Efi8Osf4Xyd+CB02Bp56NwAC49cSSfPG0M//3KOl5eVX6YKyoicnAU9AAnXge3vgbFx8Ezc+CJG2FX5xOl/vnSKUweUcg/PLWUrdW6WYmI9H8K+jbDJ8LNz8NF34B1L8N/ngZLntindZ/MifPAJ0+msaWVO554i1RruosXFBHpHxT07cXicMZtcOsCKD4efnUrPH497Nq8V7Fjigv4+lUn8OcPdvC9l9ZkqbIiIj2joO/M8GPh5udg9jfh/T/CA6fDW4/t1bq/asYorps5ih+8vJYfL3if/jZ6SUSkjYK+K7E4nP638LcLoHQK/Pqz8Ni1UL1nUvDdl5/A+ZNLuPs37/DZxxazq7ElixUWEemcgr47w46BTz8Hs++FDxfAf54Oix8FdwbkxvnRX83kK5dM5sV3tnHZ91/j7U3V2a6xiMheFPQ9EYvB6bcGrfsR02De7fCzT0D1RsyMOeccw9w5p9PUkubqH77OY29+qK4cEek3FPQHYugEuOm3cPG3Yf0bQd/9okfAnZnjhvLsHWdx+oRh/NMzb/O5uUuoa9pnRWYRkcNOQX+gYjE4bQ787eswcjr85g747ech3cqwgjx+8ulT+OKsSfxm6WYu+8FrvLt1V7ZrLCIRp6A/WEPHw1/Ng7M+D4t+DE/fDKkmYjHj9o9N5LG/Pp2axhRXPrCAp8o2dP96IiJ9REF/KGIxuOAumPV1eOfXwaicphoAzjhmGM/ecRYzRg/hH59exhd/vpSGZt28REQOPwV9bzjzdrjyv+CD1+CRy6AuWIm5pDDJz/76NO44fyK/WLyRKx54jbXlNVmurIhEjYK+t0y/EW54DMpXwsOzYWfQXROPGV+4cBKP3HwqlbXNXP6DBfzqrY436BIR6TsK+t503MXwqWegthwevggq9ixnfM6kYp6942ymjizic3OX8OVfLqexRV05ItL3dOORvrB1eTDOvrUZPvk0jNpzL4BUa5p/f3E1//XKexQX5nHB8aVcOKWEM48ZTjInnsVKi8iRbH83HlHQ95Ud6+DRq6C2Aq5/FI49f6/Dr66p4PE31/PH1RXUNbcyICfO2ROHc8HxpZw3uYTiwrwsVVxEjkQK+myp2Rq07CtWwdUPwglX71OkKdXKn9bt4PfvbOP3K7expboRM5gxejAXTCnlguNLmVhSgJll4R8gIkcKBX02NewMbmSy/g245Ntw6v/psqi7s2LzLl5aWc7vV25jeWbdnDFDB3LB8aVcMKWEU8YN1f1qRWQfCvpsa2mAn98Mq5+Hc78CH/1H6EELfUt1Ay+tLOelldtY8F4lzak0RckE5x5XwlUnH805E4uJx9TSFxEFff/Q2gLz/g6WPgGnzglWw4z1vGVe15Ti1TXb+f3Kbby0chtV9S2MKEryiY8czbUfGc244fl9WHkR6e8U9P1FOg2/+2d44wdwwjVw5Q8hkXvAL9OcSvPSym08VbaBV1ZXkHY4dfxQrps5mkumjWBgbqIPKi8i/ZmCvj9xhwXfhd/fBcMnwbEXwLizYOyZMGDIAb/c1upGfrF4Iz8v28AHlfUU5CW49MSjuHbmaE4eM1gXcUUi4pCD3sxmA98D4sD/uPs3OxzPA34KfASoBK539w/M7ELgm0Au0Ax8yd3n7+9vhT7o2yx/GhY/Ahv+DKlGwOCoE2Hc2cFj7BmQHNTjl3N3Fn5QxVNlG3h22RYaWlo5tqSA62aO4qoZozRcUyTkDinozSwOrAYuBDYCC4Eb3f2ddmU+C5zo7rea2Q3AVe5+vZnNALa5+2YzOwF4wd2P3t/fi0zQt2lphE2L4INX4f1XYeOfg4lWFoOjTgpa++POgTGnQ7KoRy9Z25Ti2WWbeapsI4s+rCIeMz42uYTrZo7m7ImamCUSRoca9GcAd7n7RZnnXwZw939rV+aFTJk3zCwBbAWKvd2LW9CHUAkc5e5NXf29yAV9Ry0NsHFhsEDa+68G2+kWsHiw/v24s2D06RDPCcqmGoNHS+Oe7XbPd9XWsLF8B+VV1cRSjXzopcz1WWxJTqAwmaAwmZP52X47h6IO+4YMzGXyiEISGtoZbXWVEE8c0LdNOTz2F/Q9uWp3NNB+QfWNwGldlXH3lJlVA8OA7e3KfAJY3FnIm9kcYA7AmDFjelClEMsZAOPPCR7nAc31QSu/Lfjf+E9Y8L39v4bFg9dJ5FGUGMCURB7HFyfZlYpx5s5X+VT696xLTucPg67k9cRp7GxK88H2emoaW9jVmKK2iztjFSUTnDOpmI9NLuGjk4oZVqDuoEh591l45m+DoL/k2zD16h4NE5bsOyzDM8xsKnAvMKuz4+7+IPAgBC36w1GnI0buQJhwbvAAaK6DbSsAg5wkJNo92p7Hc/Z5GQMGAdTvgMU/ZULZQ0zYfBefKRwJMz8DH7kJCkoAaE07tU0pahpbqGlMUdOYYuuuRl5dXcHLqyr47bItmMFJowbzscklfGxyCVNHFu258NvaEtxIffULwSNZBGd9ASZfekBDSiPBHareh4JSyO2nQ2RbU/DS3fD6/XDU9KBb8enPwNu/hI9/BwpLs11D6Uafd92Y2ShgPnCzuy/orkKR77o5XNKtsOZF+POD8N58iOXA1KuCMf6jZnbZUkunnbc3VzP/3XJeXlXBso07cYeJBc18ZsRazmURIypew5pqIJ4XfDPZsQ52vAelJ8A5X4LjL1fgAzTuCuZWvPMrwGDIWCiZCiXHQ+kUKJkCw47t9MR92NRsDSb7rX8dZt4Cs/8t+Mb4xvfh5X8LGiIXfwumXavWfZYdah99guBi7PnAJoKLsX/p7ivalbkNmNbuYuzV7n6dmQ0GXgHudvdf9qSyCvos2L4GFv4PLHkcmnYFrbZT5wRr8+QM6Px33GH7auqW/4b6t59j2I63iJGm3Afzss9gU/E5DJ12EWdPHcuEoXnYimfglW9B5ZogwM75Eky5MrqBv20FzP0UVH0Q3I4ykYTyFcH9DLavAc8sYR3LCYbhlk4JTgBtJ4LBY/o+WN//Izx9CzTXwmXfgxOv2/t4xWr49WeD60iTLoZL74Oio/q2TtKl3hheeQnwXYLhlQ+7+9fN7B6gzN3nmVkSeBSYAewAbnD3dWb2/4AvA2vavdwsdy/v6m8p6LOoqRaWzYU//wgqVsKAoXDyXwVdO0PGZrpkXofV/wurng+6HABGTINJF5OaeBFlzWN5edV25r9bzpryWgDyc+Pk5yUoyjVm2Rv8ZeNcRqXWsyV3HK+U3sR7JRcyMJlHQV6C/LwE+XlxCvISFOQlGDEoyYhBSfISIRoptORx+O0Xggua1zwM4/5i7+OppiDsy98JTgjlK4Pt6naXynILoWQyTDgvWD8p0+3WK9JpWHAfzP8aDD0mWH215PguyrbCn34I8/81+AY3+xsw/ZNHXuu+fgdsWhyctDYuhM2Lg29Ts74OYzpekuyfNGFKDox7cPH3zw8GF+BwGHVqEDhN1cF/6AkfhUmzYdJFMGhUpy+zYUc9f1hVzvvb66lrSlHXnKKuKUV9YzMzal/huvonmeDrec9Hcn/LlfwmfSbpLu6FU1yYx8jBAxg5KBn87LA9LD+XWH9f96elAZ77Erz1aDBX4pqHDyygG6uh/N0g9Mvfga1vB4vlxXPhpOvhjNuh+LhDq2P9DnjmVljzQnCx9fL7Ia+w+9+rfA9+fXvQxXPsBcE3gC4+F1nXmgq+PW1cCBsXBT8rM21RiwXfOI+aDu+9BDVbglnsF97df/89GQp6OXjVG6Hsx8FF1ZHTg7toTTi3dy4cptOwcl7QpVO+gtahx1BzyufZPv4y6lpgV2MLW6sb2byzkc07G9hc3RD83NlIQ4e7c+XGYxw1OMnIQQM4anCSYfm5DMhNMDA3zsDcOMmc+O7tATmd7U+QzIn13UziyvfgqZtg23I4+4tw3lcg1gvfUravhT89EHxLSDUGJ98z7whmWh/ov2XTYvj5TbBrC1z0jeCbwoG8RjoNC38UzPq2OFz0NTj5puy37mu27mmpbyyDzW9BS31wLL8YRp0SXJcadQqMnLHnxNZUG8xif/37gMFf/H3wyB3Y+3Vsrg+u1SQHw+RLDuolFPTSv6XT8O5vg8DfthyGTgjC8MTrg6F87kE/cd12qK/E6ypo2LmNmsotNOwsp7W2Auq2k9NYyYCWnRSmd1LjA1iWnsByH8/S9DEsT09gO/sf+20G+bkJjhqUZPTQgYwaMoBRQwYweshARg0Jng8emHPgJ4N35sGvbwtai1f/CCZ1Ovjs0NRtD7rcFv4I6ith5Mlw5t8FF77j3Qyuc4eyh+F/74T8Erjukb3uinbAdrwfXGT+4NWgUXDZ/UHX3+HSVAvr/hB8K3nv5T1dXrGcYBJi+2DvybWOnevhd1+FFc9A0dFwwd0w7ZreOYFtWwGLfgJL5wbflqdcAdf99KBeSkEvR4Z0GlY9B6/cC1uXBUMOYzlQvz2zTEQnEgMgfzgMHBb8zC8Otut34Jvfgop3MYLPeEvBSOqGTaN6yDR2DJrKtoLJ7KKQhpZW6ptbaWhOsasxxeadDWysamBDVT01jXvPKSjIS+w+AbSF/6ghAxk9dABFyRzMwMwwgNYWihZ8jYLF/01z6XSqL/sfvGh08EIGmVIkYkZhMtE7k9Ga64MVUt/4QTDaafBYOOO2oN88r6CT8nXwm8/B8qeCLperfwQDhx56PdJpWPTjICDdg66Pmbf03cX3Hetg9YvB9aMPFwSzy/OKghPNmDOCUB8xLRiCfLA+fD04GW5ZGnRlXvxNOPojB/46zXXBSWPRT4JvGfG8IOA/8umD+yaWoaCXI4t78B926ZOQMxDyh2UCfHgm1Ifv2dddF1JTbXDS2LQ4+Mq++a1gqGebIeODr+tHnxz8POqkvfqkqxta2FhVz4YdDWysqmdj1Z6fG3bUU9fc+Q3eR1DJD3K/z8zYan6SmsU3Up+kmf0PkyxKJhiSn8vggbkMGZjD0IF7tgfnd9iXn8OQgbldL2eRbg0umL9+P2x4M+gSOOUWOPVv9ox7r1gNT30quAPaeV8JvkX1dhDvXA/z7oB1L8Po04LgHTohuMg7dEJwUjmYYEs1B9cn1rwYdCu29bEPnwQTZwXXjsac0ftDU9OtQTfZS/dAXTmcdCOc/y89G220dTkseiQY8NC0K6jrR26Gk27olZOrgl6kvYaqoFXWPvx3j2gxGD4xuBg3cnoQ/iNO7LQ17O7srG/Z3fqvbUqBQ0nFAk5fcifx1ib+PO1u1o+cjTt45ptFsJ3ZAFJpp7qhhaq6ZqrqW6iqb2Zn5mdVXXOXJxOAATlxhubndvkYMjCXsfXLGf3uw+Svex7iOdiJ1wcntN/9S9DC/cRDcMx5uDstrU5za5qmllaaUunMo5WmlnRmfxrHyYnHMg/b62cis53bbjvHjNjSn8Gr/wFVH5L51weSgzLB3y78h06AYccE38zanwRqy2HN74IumbXzobkmuBA97qzg2sTEWTB0/KF8MnquqSb497zxQPCt8+zPBxfDOw5Hbq4LJpYt+glsKgta71OvDFrvY87o1esXCnqR7tRWwJYlQfhvWQKbl0DN5szBHoZ/uhX++G34wzeheHLQ11o86ZCr1pRqpbq+har6FnbUNbOzfs8JoaqumR11zezIbFfWdX1yGGtb+evE81wTf4UBNLMsNpmvxL7AhtYhQZin0vRVHMRjRk7cGJzjHJtbyTHxbYyzbYxmCyNbt1Ca2sSQlm3ESO/+neZEAQ0FY2gsGkd+/UYKti8DoLVgBH7sLOLHXYRNOLfzLqnDZcf7wT0mVv4GBo2BWfcE80O2Lg/CfdlTwQlp+HEw8+bgulNvdI11QkEvcjBqtu0J/e7Cv3RqsAbRe/PhxBvg0u9kdUmDxpZWquqbqaxtpqo+czLIPBqryxlSvYJ1BTPJyc0lLxEnLxEjNxEjLxEjLxHfs52TeR4PtnPjwcikVGvQwk+1Oi2taVrSTksqTSqdprl1z3ZL2/HWNM2pNPXNwfWQ2qYU9c0paptaqW9KUd/cSmNjA0OatzKaLYyzrYy14GQw1rZSRSHzW2fwcnoGK3wsYMRjtnuORtujIC8YQVWQmY+Rn5tgQNtoq9wEA3PatoNye7bjDMwJyuYmuu6+SqedVNppTTst6TStrcHz+IevUvCHfyZ3+zuk8keQqNuKx5M0HXc5qek3ERt7GjmJOImY9dnILgW9SG/ZX/jH8+CSb/WPIYVHsOZUeve8i7aTQt3uRyt1zal2+/Y+abSVq23a87vNqXT3f7SdRMwYkBsnHrPdQZ5Kp0mlfb/feGKkuT7+MrNjC3k5PZ1nWs+imn2/bezu5ooZuYkYiViMnESw7/zJJfzTx6cc6FsGHPrqlSLSprAUCi8KLva1qS0P+vzb+pblkOQmYuQmchmSf+C32exMqjVNQ0srDZlvE/XNrTS0pPZs796fCrYzZd2deCxGIh58e8iJ2V7PE5lHPBPaQffUDKowTkg7k9u+ybQ6qU6223/badseMaiLJUcOkYJe5FAVlMDEC7NdC+lCIh6jMB6jMJnFxeGyLKIrSomIRIeCXkQk5BT0IiIhp6AXEQk5Bb2ISMgp6EVEQk5BLyIScgp6EZGQ63dLIJhZBfDhIbzEcGB7L1UnjPT+7J/en+7pPdq/bL0/Y929uLMD/S7oD5WZlXW13oPo/emO3p/u6T3av/74/qjrRkQk5BT0IiIhF8agfzDbFejn9P7sn96f7uk92r9+9/6Ero9eRET2FsYWvYiItKOgFxEJudAEvZnNNrNVZrbWzO7Mdn36IzP7wMyWm9kSM4v8/RrN7GEzKzezt9vtG2pmvzOzNZmfQ7JZx2zr4j26y8w2ZT5HS8zskmzWMZvMbLSZvWxm75jZCjP7+8z+fvU5CkXQm1kceAC4GJgC3GhmB3fjxfA7z92n97dxvlnyE2B2h313Ai+5+0TgpczzKPsJ+75HAPdlPkfT3f25w1yn/iQF/IO7TwFOB27LZE+/+hyFIuiBU4G17r7O3ZuBJ4Erslwn6efc/Y/Ajg67rwAeyWw/Alx5OOvU33TxHkmGu29x98WZ7RpgJXA0/exzFJagPxrY0O75xsw+2ZsDL5rZIjObk+3K9FOl7r4ls70VKM1mZfqx281sWaZrJ9LdW23MbBwwA3iTfvY5CkvQS8+c5e4nE3Rx3WZm52S7Qv2ZB2OPNf54Xz8EjgGmA1uA/8hqbfoBMysAfgF8zt13tT/WHz5HYQn6TcDods9HZfZJO+6+KfOzHHiGoMtL9rbNzI4CyPwsz3J9+h133+bure6eBn5ExD9HZpZDEPKPufsvM7v71ecoLEG/EJhoZuPNLBe4AZiX5Tr1K2aWb2aFbdvALODt/f9WJM0Dbsps3wT8Oot16ZfaAizjKiL8OTIzAx4CVrr7d9od6lefo9DMjM0M8fouEAcedvevZ7dG/YuZTSBoxQMkgMej/h6Z2RPAuQTLym4D/gX4FfAUMIZguezr3D2yFyO7eI/OJei2ceAD4G/a9UdHipmdBbwKLAfSmd1fIein7zefo9AEvYiIdC4sXTciItIFBb2ISMgp6EVEQk5BLyIScgp6EZGQU9CLiIScgl5EJOT+P6FpTiwsSQpgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['mae'])\n",
    "plt.plot(history.history['val_mae'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stack RNN layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model_2():\n",
    "    metric = metrics.MAPE\n",
    "\n",
    "    model = Sequential()\n",
    "\n",
    "    normalizer = Normalization()\n",
    "    normalizer.adapt(X_train)\n",
    "    \n",
    "    model.add(normalizer)\n",
    "    model.add(layers.SimpleRNN(units=10, activation='tanh'))\n",
    "    model.add(layers.Dense(20, activation=\"relu\"))\n",
    "    model.add(layers.Dense(1, activation=\"linear\"))\n",
    "\n",
    "    model.compile(loss='mse',\n",
    "                  metrics=metric,\n",
    "                  optimizer='rmsprop') # Use `rmsprop`\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "100/100 [==============================] - 1s 9ms/step - loss: 0.0311 - mean_absolute_percentage_error: 355.3450 - val_loss: 0.0086 - val_mean_absolute_percentage_error: 180.9863\n",
      "Epoch 2/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 0.0057 - mean_absolute_percentage_error: 157.6231 - val_loss: 0.0032 - val_mean_absolute_percentage_error: 99.5424\n",
      "Epoch 3/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 0.0029 - mean_absolute_percentage_error: 107.4928 - val_loss: 0.0021 - val_mean_absolute_percentage_error: 70.8605\n",
      "Epoch 4/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 0.0020 - mean_absolute_percentage_error: 88.5217 - val_loss: 0.0015 - val_mean_absolute_percentage_error: 63.6223\n",
      "Epoch 5/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 0.0015 - mean_absolute_percentage_error: 73.2050 - val_loss: 0.0017 - val_mean_absolute_percentage_error: 67.5739\n",
      "Epoch 6/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 0.0013 - mean_absolute_percentage_error: 66.0299 - val_loss: 0.0012 - val_mean_absolute_percentage_error: 54.5383\n",
      "Epoch 7/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 0.0011 - mean_absolute_percentage_error: 61.7516 - val_loss: 0.0011 - val_mean_absolute_percentage_error: 59.4539\n",
      "Epoch 8/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 0.0011 - mean_absolute_percentage_error: 57.0297 - val_loss: 0.0011 - val_mean_absolute_percentage_error: 55.2553\n",
      "Epoch 9/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 0.0010 - mean_absolute_percentage_error: 54.2753 - val_loss: 0.0010 - val_mean_absolute_percentage_error: 48.1751\n",
      "Epoch 10/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 9.3825e-04 - mean_absolute_percentage_error: 51.5394 - val_loss: 8.7269e-04 - val_mean_absolute_percentage_error: 44.5298\n",
      "Epoch 11/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 9.2549e-04 - mean_absolute_percentage_error: 49.4717 - val_loss: 8.4474e-04 - val_mean_absolute_percentage_error: 46.5807\n",
      "Epoch 12/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 8.7556e-04 - mean_absolute_percentage_error: 47.9004 - val_loss: 7.7914e-04 - val_mean_absolute_percentage_error: 43.5881\n",
      "Epoch 13/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 8.7287e-04 - mean_absolute_percentage_error: 46.8505 - val_loss: 7.8177e-04 - val_mean_absolute_percentage_error: 41.8640\n",
      "Epoch 14/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 8.2582e-04 - mean_absolute_percentage_error: 43.9648 - val_loss: 7.6606e-04 - val_mean_absolute_percentage_error: 39.9756\n",
      "Epoch 15/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.8201e-04 - mean_absolute_percentage_error: 45.6629 - val_loss: 7.6122e-04 - val_mean_absolute_percentage_error: 38.1941\n",
      "Epoch 16/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.7404e-04 - mean_absolute_percentage_error: 42.5864 - val_loss: 7.7488e-04 - val_mean_absolute_percentage_error: 43.5204\n",
      "Epoch 17/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.9536e-04 - mean_absolute_percentage_error: 41.9668 - val_loss: 7.7697e-04 - val_mean_absolute_percentage_error: 40.9488\n",
      "Epoch 18/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.9810e-04 - mean_absolute_percentage_error: 43.4485 - val_loss: 7.8011e-04 - val_mean_absolute_percentage_error: 37.3428\n",
      "Epoch 19/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.5929e-04 - mean_absolute_percentage_error: 41.9926 - val_loss: 0.0012 - val_mean_absolute_percentage_error: 41.9715\n",
      "Epoch 20/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 7.5339e-04 - mean_absolute_percentage_error: 41.3990 - val_loss: 7.7975e-04 - val_mean_absolute_percentage_error: 44.5581\n",
      "Epoch 21/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 7.3280e-04 - mean_absolute_percentage_error: 41.8881 - val_loss: 8.3677e-04 - val_mean_absolute_percentage_error: 42.9292\n",
      "Epoch 22/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 7.2927e-04 - mean_absolute_percentage_error: 39.9351 - val_loss: 7.1040e-04 - val_mean_absolute_percentage_error: 43.0033\n",
      "Epoch 23/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.8620e-04 - mean_absolute_percentage_error: 38.9623 - val_loss: 7.4812e-04 - val_mean_absolute_percentage_error: 36.4126\n",
      "Epoch 24/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 7.0835e-04 - mean_absolute_percentage_error: 39.5751 - val_loss: 7.6159e-04 - val_mean_absolute_percentage_error: 32.2443\n",
      "Epoch 25/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.8848e-04 - mean_absolute_percentage_error: 37.6413 - val_loss: 6.4697e-04 - val_mean_absolute_percentage_error: 35.2908\n",
      "Epoch 26/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.9029e-04 - mean_absolute_percentage_error: 37.4912 - val_loss: 6.8845e-04 - val_mean_absolute_percentage_error: 42.0360\n",
      "Epoch 27/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.9104e-04 - mean_absolute_percentage_error: 36.5584 - val_loss: 6.2653e-04 - val_mean_absolute_percentage_error: 32.7311\n",
      "Epoch 28/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.8076e-04 - mean_absolute_percentage_error: 36.8006 - val_loss: 5.7877e-04 - val_mean_absolute_percentage_error: 28.7125\n",
      "Epoch 29/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.2761e-04 - mean_absolute_percentage_error: 36.3290 - val_loss: 6.2645e-04 - val_mean_absolute_percentage_error: 38.3697\n",
      "Epoch 30/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.7042e-04 - mean_absolute_percentage_error: 35.6641 - val_loss: 6.0915e-04 - val_mean_absolute_percentage_error: 35.9833\n",
      "Epoch 31/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.5401e-04 - mean_absolute_percentage_error: 35.7725 - val_loss: 6.0657e-04 - val_mean_absolute_percentage_error: 30.8082\n",
      "Epoch 32/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.5546e-04 - mean_absolute_percentage_error: 35.5953 - val_loss: 5.9882e-04 - val_mean_absolute_percentage_error: 30.2111\n",
      "Epoch 33/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.3698e-04 - mean_absolute_percentage_error: 35.3259 - val_loss: 5.9358e-04 - val_mean_absolute_percentage_error: 35.9900\n",
      "Epoch 34/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.3399e-04 - mean_absolute_percentage_error: 35.1957 - val_loss: 0.0013 - val_mean_absolute_percentage_error: 40.3479\n",
      "Epoch 35/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.3171e-04 - mean_absolute_percentage_error: 35.2046 - val_loss: 6.3405e-04 - val_mean_absolute_percentage_error: 44.4304\n",
      "Epoch 36/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.1764e-04 - mean_absolute_percentage_error: 37.0755 - val_loss: 8.4238e-04 - val_mean_absolute_percentage_error: 31.9216\n",
      "Epoch 37/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.4061e-04 - mean_absolute_percentage_error: 35.6904 - val_loss: 5.7744e-04 - val_mean_absolute_percentage_error: 32.3169\n",
      "Epoch 38/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.2714e-04 - mean_absolute_percentage_error: 35.0627 - val_loss: 7.3906e-04 - val_mean_absolute_percentage_error: 40.4563\n",
      "Epoch 39/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.1138e-04 - mean_absolute_percentage_error: 33.9360 - val_loss: 5.7046e-04 - val_mean_absolute_percentage_error: 34.3821\n",
      "Epoch 40/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.2074e-04 - mean_absolute_percentage_error: 33.8456 - val_loss: 5.7152e-04 - val_mean_absolute_percentage_error: 32.1211\n",
      "Epoch 41/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.2663e-04 - mean_absolute_percentage_error: 33.8544 - val_loss: 5.6770e-04 - val_mean_absolute_percentage_error: 35.6191\n",
      "Epoch 42/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.0929e-04 - mean_absolute_percentage_error: 34.3838 - val_loss: 5.7221e-04 - val_mean_absolute_percentage_error: 31.0354\n",
      "Epoch 43/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 6.1109e-04 - mean_absolute_percentage_error: 32.7084 - val_loss: 5.6060e-04 - val_mean_absolute_percentage_error: 31.3868\n",
      "Epoch 44/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0325e-04 - mean_absolute_percentage_error: 33.7808 - val_loss: 5.8412e-04 - val_mean_absolute_percentage_error: 32.0484\n",
      "Epoch 45/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0138e-04 - mean_absolute_percentage_error: 34.0707 - val_loss: 6.3030e-04 - val_mean_absolute_percentage_error: 40.4973\n",
      "Epoch 46/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0248e-04 - mean_absolute_percentage_error: 33.1628 - val_loss: 6.3735e-04 - val_mean_absolute_percentage_error: 32.3922\n",
      "Epoch 47/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.9436e-04 - mean_absolute_percentage_error: 32.1142 - val_loss: 8.5812e-04 - val_mean_absolute_percentage_error: 44.6665\n",
      "Epoch 48/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.9357e-04 - mean_absolute_percentage_error: 32.6411 - val_loss: 6.4715e-04 - val_mean_absolute_percentage_error: 40.5374\n",
      "Epoch 49/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.9456e-04 - mean_absolute_percentage_error: 33.9883 - val_loss: 6.5065e-04 - val_mean_absolute_percentage_error: 38.0019\n",
      "Epoch 50/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0008e-04 - mean_absolute_percentage_error: 32.9069 - val_loss: 5.5172e-04 - val_mean_absolute_percentage_error: 31.1613\n",
      "Epoch 51/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0112e-04 - mean_absolute_percentage_error: 32.7679 - val_loss: 6.0441e-04 - val_mean_absolute_percentage_error: 34.1152\n",
      "Epoch 52/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.6786e-04 - mean_absolute_percentage_error: 32.2215 - val_loss: 6.0003e-04 - val_mean_absolute_percentage_error: 38.6933\n",
      "Epoch 53/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.7079e-04 - mean_absolute_percentage_error: 31.6489 - val_loss: 6.9499e-04 - val_mean_absolute_percentage_error: 42.3669\n",
      "Epoch 54/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.7320e-04 - mean_absolute_percentage_error: 31.6299 - val_loss: 7.2650e-04 - val_mean_absolute_percentage_error: 32.7920\n",
      "Epoch 55/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.6729e-04 - mean_absolute_percentage_error: 31.5662 - val_loss: 5.5102e-04 - val_mean_absolute_percentage_error: 31.4660\n",
      "Epoch 56/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 6.0181e-04 - mean_absolute_percentage_error: 31.2955 - val_loss: 5.8070e-04 - val_mean_absolute_percentage_error: 33.1288\n",
      "Epoch 57/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.8113e-04 - mean_absolute_percentage_error: 31.6370 - val_loss: 5.4348e-04 - val_mean_absolute_percentage_error: 35.4321\n",
      "Epoch 58/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.8223e-04 - mean_absolute_percentage_error: 31.3198 - val_loss: 5.6863e-04 - val_mean_absolute_percentage_error: 34.7886\n",
      "Epoch 59/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.6365e-04 - mean_absolute_percentage_error: 31.5509 - val_loss: 5.6284e-04 - val_mean_absolute_percentage_error: 36.1930\n",
      "Epoch 60/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.5930e-04 - mean_absolute_percentage_error: 31.1051 - val_loss: 6.9620e-04 - val_mean_absolute_percentage_error: 37.7659\n",
      "Epoch 61/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.6932e-04 - mean_absolute_percentage_error: 31.3377 - val_loss: 5.4007e-04 - val_mean_absolute_percentage_error: 31.7169\n",
      "Epoch 62/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.7361e-04 - mean_absolute_percentage_error: 31.8211 - val_loss: 5.4700e-04 - val_mean_absolute_percentage_error: 31.7961\n",
      "Epoch 63/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.6409e-04 - mean_absolute_percentage_error: 30.5714 - val_loss: 5.4645e-04 - val_mean_absolute_percentage_error: 30.4767\n",
      "Epoch 64/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5773e-04 - mean_absolute_percentage_error: 31.7787 - val_loss: 5.2547e-04 - val_mean_absolute_percentage_error: 30.5335\n",
      "Epoch 65/1000\n",
      "100/100 [==============================] - 1s 8ms/step - loss: 5.5698e-04 - mean_absolute_percentage_error: 29.9975 - val_loss: 5.5766e-04 - val_mean_absolute_percentage_error: 32.9535\n",
      "Epoch 66/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.6364e-04 - mean_absolute_percentage_error: 30.4167 - val_loss: 5.9377e-04 - val_mean_absolute_percentage_error: 35.4800\n",
      "Epoch 67/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.4695e-04 - mean_absolute_percentage_error: 30.4062 - val_loss: 5.5493e-04 - val_mean_absolute_percentage_error: 32.6147\n",
      "Epoch 68/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.6070e-04 - mean_absolute_percentage_error: 32.1716 - val_loss: 5.3095e-04 - val_mean_absolute_percentage_error: 31.8869\n",
      "Epoch 69/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5821e-04 - mean_absolute_percentage_error: 30.5646 - val_loss: 6.1187e-04 - val_mean_absolute_percentage_error: 38.0847\n",
      "Epoch 70/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5447e-04 - mean_absolute_percentage_error: 29.5703 - val_loss: 5.3236e-04 - val_mean_absolute_percentage_error: 29.6450\n",
      "Epoch 71/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5202e-04 - mean_absolute_percentage_error: 30.7104 - val_loss: 5.9717e-04 - val_mean_absolute_percentage_error: 36.2362\n",
      "Epoch 72/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.6025e-04 - mean_absolute_percentage_error: 30.1782 - val_loss: 5.6610e-04 - val_mean_absolute_percentage_error: 35.5303\n",
      "Epoch 73/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5829e-04 - mean_absolute_percentage_error: 30.8716 - val_loss: 6.4400e-04 - val_mean_absolute_percentage_error: 35.8896\n",
      "Epoch 74/1000\n",
      "100/100 [==============================] - 1s 7ms/step - loss: 5.5788e-04 - mean_absolute_percentage_error: 28.8169 - val_loss: 5.4837e-04 - val_mean_absolute_percentage_error: 32.9582\n",
      "13/13 [==============================] - 0s 3ms/step - loss: 6.6140e-04 - mean_absolute_percentage_error: 30.7619\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0006613968289457262, 30.761852264404297]"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = init_model_2()\n",
    "\n",
    "es = EarlyStopping(patience=10, restore_best_weights=True)\n",
    "\n",
    "history = model.fit(X_train, y_train,\n",
    "              epochs=1000,  # Use early stop in practice\n",
    "              batch_size=16, \n",
    "              verbose=1,\n",
    "              callbacks=[es],\n",
    "              validation_split=0.2)\n",
    "\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subsample in sequence of different lenghts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df.iloc[0:round(len(df) * 0.8)]\n",
    "df_test = df.iloc[round(len(df) * 0.8):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsample_sequence(df, length):\n",
    "    value = df.sample(1).index[0]\n",
    "    idx_position = df.index.get_loc(value)\n",
    "    if len(df) - length < idx_position:\n",
    "        return subsample_sequence(df, length)    \n",
    "    return df.loc[value:value+length-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-53-d8618f716e00>:19: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return np.array(X), np.array(y)\n"
     ]
    }
   ],
   "source": [
    "def get_X_y(df, sequence_lengths):\n",
    "    for idx, length in enumerate(sequence_lengths):\n",
    "        df_subsample = subsample_sequence(df, length)\n",
    "        X_subsample = df_subsample.drop(columns='pm2.5')\n",
    "        y_subsample = df_subsample['pm2.5']\n",
    "        if y_subsample.isna().sum() == 0:\n",
    "            pass\n",
    "        elif y_subsample.isna().sum() == length:\n",
    "            y_subsample.fillna(df['pm2.5'].mean(), inplace=True)\n",
    "        else:\n",
    "            y_subsample.fillna(y_subsample.mean(), inplace=True)\n",
    "        if idx == 0:\n",
    "            X =[np.array(X_subsample.to_numpy().tolist())]\n",
    "            y =[np.array(y_subsample.to_numpy().tolist())]\n",
    "        else:\n",
    "            X.append(np.array(X_subsample.to_numpy().tolist()))\n",
    "            y.append(np.array(y_subsample.to_numpy().tolist()))\n",
    "            \n",
    "    return np.array(X), np.array(y)\n",
    "\n",
    "sequence_lengths_train = np.random.randint(15, 30, 1000)\n",
    "X_train, y_train = get_X_y(df_train, sequence_lengths_train)\n",
    "\n",
    "sequence_lengths_test = np.random.randint(20, 30, 200)\n",
    "X_test, y_test = get_X_y(df_test, sequence_lengths_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_pad = pad_sequences(X_test, dtype='float32', padding='post', value=-1000, maxlen=max(sequence_lengths_train))\n",
    "y_test_pad = pad_sequences(y_test, dtype='float32', padding='post', value=-1000, maxlen=max(sequence_lengths_train))\n",
    "\n",
    "X_train_pad = pad_sequences(X_train, dtype='float32', padding='post', value=-1000, maxlen=max(sequence_lengths_train))\n",
    "y_train_pad = pad_sequences(y_train, dtype='float32', padding='post', value=-1000, maxlen=max(sequence_lengths_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pad =np.expand_dims(y_train_pad, axis=2)\n",
    "y_test_pad = np.expand_dims(y_test_pad, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model():\n",
    "    ### YOUR CODE HERE\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(layers.Masking(mask_value=-1000, input_shape=(29,6)))\n",
    "    model.add(LSTM(units=10, activation='tanh', return_sequences=True))\n",
    "    model.add(layers.Dense(20, activation=\"relu\"))\n",
    "    model.add(layers.Dense(1, activation=\"linear\"))\n",
    "\n",
    "    model.compile(loss='mse',\n",
    "                  metrics=['mae'],\n",
    "                  optimizer='rmsprop') # Use `rmsprop`\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "13/13 [==============================] - 4s 78ms/step - loss: 0.2072 - mae: 0.3618 - val_loss: 0.1113 - val_mae: 0.2603\n",
      "Epoch 2/1000\n",
      "13/13 [==============================] - 0s 13ms/step - loss: 0.0822 - mae: 0.2231 - val_loss: 0.0507 - val_mae: 0.1785\n",
      "Epoch 3/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0382 - mae: 0.1542 - val_loss: 0.0238 - val_mae: 0.1241\n",
      "Epoch 4/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0179 - mae: 0.1080 - val_loss: 0.0119 - val_mae: 0.0907\n",
      "Epoch 5/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0094 - mae: 0.0808 - val_loss: 0.0079 - val_mae: 0.0728\n",
      "Epoch 6/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0067 - mae: 0.0681 - val_loss: 0.0069 - val_mae: 0.0677\n",
      "Epoch 7/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0059 - mae: 0.0641 - val_loss: 0.0065 - val_mae: 0.0655\n",
      "Epoch 8/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0055 - mae: 0.0621 - val_loss: 0.0057 - val_mae: 0.0632\n",
      "Epoch 9/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0052 - mae: 0.0603 - val_loss: 0.0056 - val_mae: 0.0607\n",
      "Epoch 10/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0049 - mae: 0.0591 - val_loss: 0.0054 - val_mae: 0.0602\n",
      "Epoch 11/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0047 - mae: 0.0574 - val_loss: 0.0053 - val_mae: 0.0588\n",
      "Epoch 12/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0046 - mae: 0.0568 - val_loss: 0.0051 - val_mae: 0.0597\n",
      "Epoch 13/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0044 - mae: 0.0553 - val_loss: 0.0052 - val_mae: 0.0590\n",
      "Epoch 14/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0044 - mae: 0.0553 - val_loss: 0.0051 - val_mae: 0.0565\n",
      "Epoch 15/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0043 - mae: 0.0541 - val_loss: 0.0050 - val_mae: 0.0580\n",
      "Epoch 16/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0042 - mae: 0.0543 - val_loss: 0.0050 - val_mae: 0.0554\n",
      "Epoch 17/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0041 - mae: 0.0529 - val_loss: 0.0050 - val_mae: 0.0610\n",
      "Epoch 18/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0042 - mae: 0.0540 - val_loss: 0.0047 - val_mae: 0.0564\n",
      "Epoch 19/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0041 - mae: 0.0528 - val_loss: 0.0048 - val_mae: 0.0557\n",
      "Epoch 20/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0041 - mae: 0.0528 - val_loss: 0.0048 - val_mae: 0.0544\n",
      "Epoch 21/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0041 - mae: 0.0530 - val_loss: 0.0047 - val_mae: 0.0543\n",
      "Epoch 22/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0040 - mae: 0.0523 - val_loss: 0.0046 - val_mae: 0.0553\n",
      "Epoch 23/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0040 - mae: 0.0520 - val_loss: 0.0046 - val_mae: 0.0541\n",
      "Epoch 24/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0040 - mae: 0.0521 - val_loss: 0.0046 - val_mae: 0.0565\n",
      "Epoch 25/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0039 - mae: 0.0519 - val_loss: 0.0047 - val_mae: 0.0584\n",
      "Epoch 26/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0040 - mae: 0.0524 - val_loss: 0.0045 - val_mae: 0.0539\n",
      "Epoch 27/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0039 - mae: 0.0516 - val_loss: 0.0046 - val_mae: 0.0532\n",
      "Epoch 28/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0512 - val_loss: 0.0049 - val_mae: 0.0551\n",
      "Epoch 29/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0039 - mae: 0.0514 - val_loss: 0.0048 - val_mae: 0.0536\n",
      "Epoch 30/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0038 - mae: 0.0507 - val_loss: 0.0045 - val_mae: 0.0537\n",
      "Epoch 31/1000\n",
      "13/13 [==============================] - 0s 9ms/step - loss: 0.0038 - mae: 0.0507 - val_loss: 0.0046 - val_mae: 0.0536\n"
     ]
    }
   ],
   "source": [
    "model = init_model()\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=5, restore_best_weights=True)\n",
    "\n",
    "history = model.fit(X_train_pad, y_train_pad,\n",
    "            validation_split=0.2,\n",
    "            epochs=1000, \n",
    "            batch_size=64,\n",
    "            callbacks=[es], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f14ec2a6640>]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnLUlEQVR4nO3deZgcd33n8fe3u+ceaQ7NSJoZ3dLItyXjwdwOh7DlADIkQOwsidknzzosOJDw7G6cC4h52OWBxEmWdQAnkA1JwBjMIYyzxoA5HDBobMs2ki1LlmXd0mjus8/v/lE1o55DUssaqWeqP6/H/VT3r6q6fzVtfar6V7/6lbk7IiISXbFiV0BERM4vBb2ISMQp6EVEIk5BLyIScQp6EZGISxS7AlM1NTX5qlWril0NEZF55bHHHjvh7s0zzZtzQb9q1So6OzuLXQ0RkXnFzF481Tw13YiIRJyCXkQk4hT0IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScZEJ+v7RNH/3/d08eaCv2FUREZlTIhP0AH/z/ef4xQvdxa6GiMicEpmgX1iZoLYiweG+sWJXRURkTolM0JsZLXWVHO4bLXZVRETmlMgEPUBrfRVH+nVELyKSL2JBryN6EZGpCgp6M9tsZrvMbI+Z3T7D/PeZ2dNmtt3MHjGzS8PyVWY2GpZvN7PPzfYG5Gutq6J7OMVYOns+P0ZEZF454zDFZhYH7gLeDBwEtpnZVnffmbfYl939c+HyW4A7gc3hvOfdfeOs1voUWuqrADjSP8bqppoL8ZEiInNeIUf01wB73H2vu6eAe4Ab8xdw94G8lzWAz14VC9daXwnAETXfiIhMKCTo24ADea8PhmWTmNkHzOx54FPAB/NmrTazJ8zsx2b2upk+wMxuNbNOM+vs6uo6i+pP1loXHNEfUtCLiEyYtZOx7n6Xu68F/hj487D4CLDC3a8CPgx82cwWzrDu3e7e4e4dzc0z3gmrIEvrwiN69bwREZlQSNAfApbnvV4Wlp3KPcDbAdw96e7d4fPHgOeB9S+ppgWoLIvTVFuunjciInkKCfptQLuZrTazcuAmYGv+AmbWnvfyLcDusLw5PJmLma0B2oG9s1HxU2mtr+KwjuhFRCacsdeNu2fM7DbgQSAOfNHdd5jZHUCnu28FbjOzTUAa6AVuCVe/FrjDzNJADnifu/ecjw0Z11JXyd6u4fP5ESIi88oZgx7A3R8AHphS9pG85x86xXr3AfedSwXPVktdFY/sPoG7Y2YX8qNFROakSF0ZC9BWX8VwKsvAWKbYVRERmRMiF/QtYV96nZAVEQlELuhbJ66OVdCLiEAUg37ioin1vBERgQgGffOCChIx0zAIIiKhyAV9PGYsWajhikVExkUu6CHoeaOLpkREApEM+hbdgEREZEIkg761vopjA2Nkc0UZLVlEZE6JZtDXVZLOOieGksWuiohI0UUz6MO+9Gq+ERGJaNC31I0HvU7IiohEMujbdHWsiMiESAb9wqoE1eVx3VJQRISIBr2Z0VpfxRE13YiIRDPoIbgByWE13YiIRDfo2+qrdDJWRIQIB31LXRUnhpIkM9liV0VEpKgiG/St4Q1IjmrMGxEpcREOevWlFxGBkgh6nZAVkdIW2aBvqQuabnTRlIiUuoKC3sw2m9kuM9tjZrfPMP99Zva0mW03s0fM7NK8eX8SrrfLzK6fzcqfTmVZnEU15bqloIiUvDMGvZnFgbuAG4BLgZvzgzz0ZXe/wt03Ap8C7gzXvRS4CbgM2Az8ffh+F0RLfaWO6EWk5BVyRH8NsMfd97p7CrgHuDF/AXcfyHtZA4wPBH8jcI+7J939BWBP+H4XRGtdldroRaTkFRL0bcCBvNcHw7JJzOwDZvY8wRH9B89m3fNFwyCIiMziyVh3v8vd1wJ/DPz52axrZreaWaeZdXZ1dc1WlWitr2QwmWFgLD1r7ykiMt8UEvSHgOV5r5eFZadyD/D2s1nX3e929w5372hubi6gSoUZH5deR/UiUsoKCfptQLuZrTazcoKTq1vzFzCz9ryXbwF2h8+3AjeZWYWZrQbagV+ee7ULo770IiKQONMC7p4xs9uAB4E48EV332FmdwCd7r4VuM3MNgFpoBe4JVx3h5ndC+wEMsAH3P2CDT4zPgyCRrEUkVJ2xqAHcPcHgAemlH0k7/mHTrPuJ4BPvNQKnovFCyqJx0xH9CJS0iJ7ZSxAPGYsXVipNnoRKWmRDnoIhkLQLQVFpJRFPuhb66s4oqGKRaSERT7ox4dByOX8zAuLiERQ5IO+rb6KdNY5MZwsdlVERIoi8kE/ftGUbkAiIqUq8kE/3pf+iE7IikiJin7Qh0f06nkjIqUq8kFfX11GVVlcPW9EpGRFPujNjJb6Sl0dKyIlK/JBD0HPm8M6oheRElUSQd9SV6mTsSJSskoi6Fvrq+gaSpLK5IpdFRGRC640gr6uCnc4NqDmGxEpPaUR9PXqYikipaskgr5l/KIp3YBEREpQSQR9q4ZBEJESVhJBX1Uep6G6TH3pRaQkRSvosxlIz3zU3lpfpaAXkZIUnaDvOwD/qw2e/tqMs1vqdAMSESlN0Qn6hW0QS8CRJ2ec3VavWwqKSGmKTtDHYrD0ylMGfUt9FYNjGQbH0he4YiIixRWdoAdo2QBHn4Zcdtqs8b70ar4RkVJTUNCb2WYz22Vme8zs9hnmf9jMdprZU2b2AzNbmTcva2bbw8fW2az8NC0bIDMKJ3ZPm9VaF/Sl1wlZESk1Zwx6M4sDdwE3AJcCN5vZpVMWewLocPcrga8Dn8qbN+ruG8PHllmq98xaNgTTGZpvxo/o1ZdeREpNIUf01wB73H2vu6eAe4Ab8xdw94fdfSR8+SiwbHarWaCm9ZConDHoFy+oIGa6OlZESk8hQd8GHMh7fTAsO5XfA/4973WlmXWa2aNm9vaZVjCzW8NlOru6ugqo0inEE7Dkcjj61LRZiXiMpQvV80ZESs+snow1s/cAHcCn84pXunsH8NvA35rZ2qnrufvd7t7h7h3Nzc3nVomWDcERfW76kMQt9VUcUdONiJSYQoL+ELA87/WysGwSM9sE/Bmwxd2T4+Xufiic7gV+BFx1DvU9s5YNkByAvn3TZrXWV3FYTTciUmIKCfptQLuZrTazcuAmYFLvGTO7Cvg8QcgfzytvMLOK8HkT8Bpg52xVfkanOyFbV8mR/jFyOT+vVRARmUvOGPTungFuAx4EngHudfcdZnaHmY33ovk0UAt8bUo3ykuATjN7EngY+KS7n9+gX3wJxMpO2fMmlcnRPZw6r1UQEZlLEoUs5O4PAA9MKftI3vNNp1jvZ8AV51LBs5aoCMJ+hqBvqTs5Ln3zgooLWi0RkWKJ1pWx48ZPyPrkJhr1pReRUhTdoB/phoHJ54xPBr1OyIpI6Yho0G8MplOabxqqy6hIxHTRlIiUlGgG/ZLLwGLTgt7MaKuvUtONiJSUaAZ9eTU0XTTzCdn6SvWlF5GSEs2gh5MnZKdordMtBUWktEQ76AePwOCxycX1VRwfTJLOTh8iQUQkiqId9DBtgLO2+krc4ahuQCIiJSK6Qb80vE7ryPZJxS11utOUiJSW6AZ95UJoXDutnV596UWk1EQ36GHGE7Kt9eEtBdXzRkRKRPSDvm8/jPRMFFWXJ2iqLWdv13ARKyYicuFEP+hh2gnZjcvrefzF3iJUSETkwiuNoJ/SfHP1ykb2nhimeyg5w0oiItES7aCvboS6FdOCvmNVAwCdOqoXkRIQ7aAHaLlyWtBf0VZHeTzGYwp6ESkBJRD0G6F7D4wNTBRVlsW5Ylkdnft6Tr2eiEhElEDQXxlMj/1qUnHHygaePtTPWDpbhEqJiFw4JRD0M5+Q7VjVSDrrPHWwvwiVEhG5cKIf9AuWQu2SGXrejJ+QVfONiERb9IMeZrxCtrGmnDXNNTy2TydkRSTaSifou3ZBevKwBx0rG+h8sZdczk+xoojI/Fc6Qe9ZOLZzUnHHqkb6R9M83zVUpIqJiJx/BQW9mW02s11mtsfMbp9h/ofNbKeZPWVmPzCzlXnzbjGz3eHjltmsfMEmTshun1TcsVIXTolI9J0x6M0sDtwF3ABcCtxsZpdOWewJoMPdrwS+DnwqXLcR+CjwCuAa4KNm1jB71S9Q3XKoapjWTr+6qYZFNeVsU396EYmwQo7orwH2uPted08B9wA35i/g7g+7+0j48lFgWfj8euAhd+9x917gIWDz7FT9LJjNeELWzLh6ZYOukBWRSCsk6NuAA3mvD4Zlp/J7wL+fzbpmdquZdZpZZ1dXVwFVeglaNsDxnZBJTSruWNXAi90jdA1qgDMRiaZZPRlrZu8BOoBPn8167n63u3e4e0dzc/NsVumklg2QTUHXs5OKr17ZCMBj6k8vIhFVSNAfApbnvV4Wlk1iZpuAPwO2uHvybNa9IFo2BtMpzTeXty2kIhFjm/rTi0hEFRL024B2M1ttZuXATcDW/AXM7Crg8wQhfzxv1oPAdWbWEJ6EvS4su/AaVkP5gmlBX5GIs2FZvXreiEhknTHo3T0D3EYQ0M8A97r7DjO7w8y2hIt9GqgFvmZm281sa7huD/Bxgp3FNuCOsOzCi8VmHLIY4OpVDew41M9oSgOciUj0JApZyN0fAB6YUvaRvOebTrPuF4EvvtQKzqqWDdD5T5DLQiw+UdyxsoHP5pztB/p41dpFRaygiMjsK40rY8e1bIDMKJzYPal4fIAznZAVkSgqvaCHac039dXltC+uVTu9iERSaQX9onZIVM3YTt+xKrhwSgOciUjUlFbQxxOw9PKZg35lI4NjGZ47PliEiomInD+lFfQQNN8cfQpyuUnFHavCAc7Un15EIqY0gz45AL0vTCpe0VhNU22Fxr0RkcgpzaCHGQc4e/mqBo1kKSKRU3pB33wJxMrg8OPTZl29soGDvaMcGxgrQsVERM6P0gv6RDmseCXs+eG0WR2rggHO1E4vIlFSekEP0H4dHN8BfQcmFV/WupDKshidunBKRCKkNIN+/fXBdPf3JhWXxWNsXF6vI3oRiZTSDPqm9VC/clrQQ9CffueRAYaTmSJUTERk9pVm0JvB+s2w98eQHp006+pVDWRzzpMH+opTNxGRWVaaQQ+w/rpggLMXfjqp+GUrGjBDNyIRkcgo3aBf+Vooq4bdk++DUldVxkVLFuiErIhERukGfVklrHk9PPc98MkDmV29soEn9veR1QBnIhIBpRv0EHSz7N8/7YbhHasaGEpm2HVUA5yJyPynoAd4bnLzTcfK8MIpNd+ISASUdtDXtcGSK6Z1s1zWUMWShRXqTy8ikVDaQQ9B75v9j8LoyVA3MzpWNmokSxGJBAX9+s3gWdjzg0nFHasaONQ3yuG+0VOsKCIyPyjo266G6kXTmm9OttPrqF5E5jcFfSwO6zbB7ocgl50ovqRlAdXlcTo1Pr2IzHMFBb2ZbTazXWa2x8xun2H+tWb2uJllzOydU+ZlzWx7+Ng6WxWfVe3XwWgPHHpsoigRj3HN6kYe3nUcd/WnF5H564xBb2Zx4C7gBuBS4GYzu3TKYvuB9wJfnuEtRt19Y/jYco71PT/WvQksPq2b5VuuaOFAzyjbNe6NiMxjhRzRXwPscfe97p4C7gFuzF/A3fe5+1NAbqY3mPOqGmD5K6YNh3D95Uspj8f4zpNHilQxEZFzV0jQtwH5d+g4GJYVqtLMOs3sUTN7+0wLmNmt4TKdXV1dZ/HWs2j9dXD0aRg4PFG0sLKM11/UzP1PHdZwCCIyb12Ik7Er3b0D+G3gb81s7dQF3P1ud+9w947m5uYLUKUZrN8cTKf0vtmysZXjg0l++YJOyorI/FRI0B8Clue9XhaWFcTdD4XTvcCPgKvOon4XTvPFULdiWjv9Gy9eTHV5nK1PHj7FiiIic1shQb8NaDez1WZWDtwEFNR7xswazKwifN4EvAbY+VIre16ZBc03e38E6bGJ4uryBJsuWcK//+oI6ez8PAUhIqXtjEHv7hngNuBB4BngXnffYWZ3mNkWADN7uZkdBN4FfN7MdoSrXwJ0mtmTwMPAJ919bgY9QPv1kB6BFx+ZVLxlQyt9I2ke2XOiSBUTEXnpEoUs5O4PAA9MKftI3vNtBE06U9f7GXDFOdbxwln9OkhUBWPUr9s0Ufy69U0srEzwne2HecNFi4tYQRGRs6crY/OVVcHqa4NulnkXSVUk4txweQvf23mMsXT2NG8gIjL3KOinWn8d9O6DE7snFb9tQytDyQwPP3u8OPUSEXmJFPRTtV8fTKdcPPXKNY001ZbznafU+0ZE5hcF/VT1y2HxZdO6WSbiMd5yRQs/eOY4Q8lMkSonInL2FPQzWX8d7P85jPVPKn7bhlaSmRwP7TxapIqJiJw9Bf1M2q+HXAae/+Gk4petaKCtvkpj34jIvKKgn8myl0NlfdDNMk8sZrz1yhZ+8lwXvcOp4tRNROQsKehnEk8E/ej3PAS5yVfDvm1DK5mc8/92qPlGROYHBf2prL8ehrvg8BOTii9rXciaphq2blfvGxGZHxT0p7JuE1hsWjdLM+OtG1p59IVujg+MnWJlEZG5Q0F/KtWNsOLV8ORXIJueNGvLhhbc4btP66SsiMx9CvrTefVt0LcffnXfpOJ1ixdwSctCDV0sIvOCgv501m+GJZfDT++cdlJ2y4ZWntjfx4GekSJVTkSkMAr60zGD1/4RnNgFz94/adZbr2wB0JAIIjLnKejP5LJ3QOMa+OlfTxrRcnljNVetqNfFUyIy5ynozyQWD47qj2yfdqXslg2tPHNkgD3HB4tTNxGRAijoC3HlTbCwLTiqz/OWK1qIGWzVUb2IzGEK+kIkyuHVH4QX/wNe/PlE8eKFlbxyzSLuf/IwntesIyIylyjoC/Wy34XqRfDInZOK37ahlb0nhtlxeKBIFRMROT0FfaHKq+GV74fd34MjT04Ub75sKYmY8R31qReROUpBfzau+S9QsTDoVx9qqCnn19Y3c9/jBxkcS59mZRGR4lDQn43KuiDsd3570j1l/+BN7ZwYSvGZH+4pYuVERGZWUNCb2WYz22Vme8zs9hnmX2tmj5tZxszeOWXeLWa2O3zcMlsVL5pXvh8SlfDI30wUbVxez7s7lvHFR15gz/GhIlZORGS6Mwa9mcWBu4AbgEuBm83s0imL7QfeC3x5yrqNwEeBVwDXAB81s4Zzr3YR1TTB1e+Fp74ajIMT+h+bL6aqPM4d9+9UDxwRmVMKOaK/Btjj7nvdPQXcA9yYv4C773P3p4DclHWvBx5y9x537wUeAjbPQr2L69V/ABj87DMTRU21FfzRpvX85LkuHtp5rHh1ExGZopCgbwMO5L0+GJYV4lzWnbvq2mDjzfD4l2Do+ETx77xqJe2La/n4d3cyls4WsYIiIifNiZOxZnarmXWaWWdXV1exq1OY1/whZFPw87smisriMf5yy2Uc6BnlH36yt3h1ExHJU0jQHwKW571eFpYVoqB13f1ud+9w947m5uYC37rIFq0NBjzb9gUY7Z0ofvW6Jn79iqXc9aM9HOobLWIFRUQChQT9NqDdzFabWTlwE7C1wPd/ELjOzBrCk7DXhWXR8NoPQ2oQfvkPk4r/9NcvAeB/PvBMMWolIjLJGYPe3TPAbQQB/Qxwr7vvMLM7zGwLgJm93MwOAu8CPm9mO8J1e4CPE+wstgF3hGXRsPRyWH8DPPr3kDzZrXJZQzX/9dfW8d2njvCz508UsYIiImBzrStgR0eHd3Z2FrsahTuwDb6wCa77RHDrwdBYOsumO39MTXmC737wtSTic+J0iIhElJk95u4dM81T+pyr5S+H1dfCTz4FXbsmiivL4vzFWy9l17FB/vXRF4tYQREpdQr62bDlMxCvgH/9TRg4OTb9dZcu4XXtTdz50HN0DyWLWEERKWUK+tnQsAre8/Wg982//iaM9gFgZnz0bZcxksryV9/bddq3EBE5XxT0s6VlA/zWv8CJ5+Cr74FMcAS/bnEt//k1q7hn2wGeOthX3DqKSElS0M+mtW+Et38W9v0Uvvn7kAtGhPjgm9pZVFPBR7fuIJebWye/RST6FPSz7cp3wZs/Dju+CQ/+KbizoLKM22+4mCf29/HNJwq91kxEZHYo6M+HV/9BMJzxLz4LP/vfAPzGVW1ctaKeTzzwDI/v7z3DG4iIzB4F/flgFvSrv+w34KGPwJNfJRYz/updG6itSHDT5x/lK7/cf+b3ERGZBQr68yUWg3d8Dla9Dr79fnj+h6xtrmXrba/hlWsX8SffeJo/+cbTJDMa5VJEzi8F/fmUqICb/g2aL4av/g4c3k59dTn/9N6X84E3rOUrv9zPTXc/ytH+sWLXVEQiTEF/vlXWwX/6OlQ1wL+9C3peIB4z/vv1F/O597yM544O8tbPPMK2fdEZAkhE5hYF/YWwsAXec18wfv0/3QDf/xgcfIzNly3lWx94DQsqE9x896N86ef7dBtCEZl1CvoLpfki+J1vBtOffQb+8Y3wN5fR/tgd3P+2HG9c38hHvr2D//a1p3R3KhGZVRq9shhGeuC5B+HZ+2HP9yEzhlc1snPBa7jz4Hp6l76Wz/zuq2irryp2TUVknjjd6JUK+mJLDcOeH8Az3wnCP9nPsFfyc7uSdONFLGy7iJXrLqdt7eVYTVPQdVNEZIrTBX3iQldGpiivgUu3BI9MCvb9hOwT32Tjcz+koaeTeE8Ong4WHYnVMFq7kvLF66htWY8tWguNa6FxDWgnICKnoKCfSxLlsG4TC9dtAsAzSQ7t28WeZ5+ie/8zZE88z+LeQ6zqe5TqPfcTJzexaq58AbZoLda4Jrif7fgOYNFaqF6knYBICVPQz2GWqKBt3ZW0rbsSAHfnxe4Rfr63m23PH+XFvc+yYHg/q+woqzJHWT12jLVH/4MW/xaxvJ1ApnwB3rCGxOL1WNN6WLQOmtYHO4EynQcQiTq10c9j7s7B3lH294xwoGeEA70jHOgZ5UhPP9me/dSN7me1HWWlHWW1HWVN7AjL7OQ9bB1jqLKFsbo1sKidipaLqWm9mHhtMyQqgwu+ElXhtBLiOi44o2wGXnwkmK58NZRXF7tGUiLURh9RZsbyxmqWN84cJiOpDAd7RznQM8LenhH+Y2CM7t4+rGcvNQN7aRh9kVXDh1gzcpA1R7dRs/P0d8HKWZxcrBwf3wlU1JFYuASrXQzjj5opz2uaIJaAzBiMDUByIJz2T3k9ECxTVh38yiirDs5flFVBWTgtrw7KKxbCgpZgmIm5wB0OPQ5P3wu/+gYMHw/KE5XBEBjt10H7m6FxdXHrKSVLR/QlzN3pHUlzpH+Uo32j9B9/kfTx50gP9zI2MkI6OUI6OUo2NUouPUY5aSpJURFOF9oIzdbPklg/i+inipmHcvB4OZZNnblC8fLgorJCJKqgaR00XRQ0QzW1F9Yclc3AcBcMHYOh48F0uAsWLA2ucWi6CCpqC6vDiT1BuD/9NejZG9xOcv31cMW7gp3S7u/D7u9Bz/PB8ovaT4b+ylcHO8tS5x7cma3/APQfhL4D4fMDYDFY84bg77Wwtdg1nfPUvVLOWTbnDIym6RlJ0Tucons4RddgkqP9YxwdGONo/xh9/b1kB45Rleqh2fpptj6arJ8K0gx6NUNWzVishmSilmS8llSilkzZAjKJWrJltVRXlrO4NkFrDbRUO0sqszRV5mgqz9BQlqY8l4T0cHCrxu7ng7t5nXgO+vYD4/8fG9SvCEK/YRWkhqaE+om8ZU+hbgUsvjgI/uZLgrGKmsMdwOBR+NV9QbgffiL4vNXXBuF+ydugqn76+3U/D7sfCkJ/3yOQTQa/Utb8Gix7OVQsCH69lNcE5eU1wY6ivPbkL5vyGojFZ/EbvYCyaeh5Abp3w4nd0LsvCPXxcE8NTV4+UQl1y4Oux4OHg7Ill8O6TcGOcvk1EC+74Jsx1yno5YIaHEtzbGCMo/1JjvSP0juSIpnOkczkSGayjKUnT5OZHGPpLEPJDF2DSU4MpcjOcCeuuqoyFi+oYFFtOYbhOO6Q8CSt6UO0ZvbTmjlIW+YArdkDLMkeYzRWw3B5E2MVTaSrmvGaxdiCJZTVLaWqoYXqRa3ULVpKYugoua5n4fiz0PUs1vUs1r170i+RXG0LNnwM8xy0bIQr3x0MRb2wpfA/TmoYXvhpEPq7H4L+QoerNqhuhOomqGmGmkXhtDnoVVXTHDSTVTXm7RDCnlYTPa7yXnsOkoMw1n/6R3Ig2NlUL8p7NE5+XdMUjOU02hsE+Xigd+85Geyed7V3VSPULw/CvG55+HxZ+HzFyV5i7nB8Z7iTfAgOPAq5TNB0t/YNsO7NQfifzd9/2vcxAgOHwh1P+Bg8ApULTzZD1jRD7ZLgefWiObvDPeegN7PNwN8BceAf3f2TU+ZXAF8Crga6gd9y931mtgp4Bhi/M/aj7v6+032Wgl6yOadnOMXxwTGODyTzpsHz3uE0jmMY4X+YgWHBNHyec2dgLE3vcJrekRQjqbMbWiJOlhV2nHY7SLsdYm3sMAe8mR/Er2WgdjVNtcFOZ1FtBU21FTTVlgdlNeXEY0bfSJq+0TR9Iyn6R9PTXw+nKMuOsLgqx+KKLM0VaRaVZ2ksS1GfSFOXSLEglqI2lqQqN0zZWA/xsR4SoydIjHYTH+smPja7N7Fxi2GVdcFgfJV1QaimR2CkO7iiOzlQ4B+vImhGW7QuaFZb1B5O1838q6cQY/2w98fBTnLP94NAhqArcWVd3vmdqsnnesbLYvHgF1n+r4mR7ikfYkGYp4aCc0ZTWSyYX7sk2MHFy4MdEh7sPKc9J5hO/I8ZD97DYkF9xp+PPxatgzf9xUv685xT0JtZHHgOeDNwENgG3OzuO/OWeT9wpbu/z8xuAt7h7r8VBv397n55oZVV0Mv5MpbO0juSmgj+nuEUfSMpeobT5NyJmRGPBSe5Y2bEDGIW7DziMcOAoWSGE0MpTgwl6R5K0T0c/ALpHUlxun9KMQt+kdRXl4fTMuqryojHYuFOIBXuBNL0j6ZIZwv7pR0nSwNDLLJ+Gm2QBgaDHR+OhU1UJ6+gOFkGMEQVA17DANUT02EqKU/EqSmPU12eoLo8Tlk8RiJuxGNGBRnqGKTBBqnzQRbm+iemY7EajpYv52jZMrrji8lZPPybBL+88rdo/G8dMyMWM+Ljf++J58EvtlQmN/GLL5k5+aswmcqyLL2XjnQnF+eepzaWotpSVFuSSlJUkqTCk5TlkpTlxia6G2cS1YxWtzFW3UKqppVUbSvp2jayC9rILWiDBS3EyiqCcE4NEhs+QXzkOPGRLuKjJ0iMHA93tCcoGztBzLPEYobF4uG2xIKHxcJfUnbyF1UuG+4AcsEvnPHnubyyJZfBu/5vQd/9VOfa6+YaYI+77w3f7B7gRmBn3jI3Ah8Ln38d+D9mukJH5pbKsjgtdVW01M3+tQOZbI7ekTQnhpKcGEqSc2ioLqO+qpy66jIWVCSIxQr7J+HujKSy9I2m6R0Ojv4HxzK4O06QQbmJ5+Mh6uP3op8I5UTMiMdixGMQj8XC10G5mZFMZxlJZRlOZYJpMnPydTI7UZbJ5cjmnEzOyebi9ObK6c41knEn6+PlOciBZYJtNEuGUzv5iyv8peU42VxQ92wueA/34JdcNudBebjXrCyLU5GIUZGIU1kWo6osTn1VGRVlMSoSL+NAooODZoykMgyNZRhMBtOhZPAYTKZJZ3OUkaWMDCNUwNBM30MOOBA+ZlIfPtYV9B2WxY2qsjhV4Q4zHrOJkWkndnh+cjI+75LkQj5b0CecnUKCvo3JW38QeMWplnH3jJn1A4vCeavN7AlgAPhzd//p1A8ws1uBWwFWrFhxVhsgMhck4jGaF1TQvODce9KYGTUVCWoqEhrYbhYkM1mGxjIMJ7OkczkyWSedzZHJOZlsjnTWyeSVZ3Me7pjstL/yDCOVzTGayjAa7jRHw8dIevLz3Pg5J5s0Yfx4eHxnuPIUXaXP1fnuR38EWOHu3WZ2NfAtM7vM3Sc19Ln73cDdEDTdnOc6iUgJqUjEqaiNs6jAXrNRVMgVJ4eA5Xmvl4VlMy5jZgmgDuh296S7dwO4+2PA88D6c620iIgUrpCg3wa0m9lqMysHbgK2TllmK3BL+PydwA/d3c2sOTyZi5mtAdqBvbNTdRERKcQZm27CNvfbgAcJuld+0d13mNkdQKe7bwW+APyLme0Begh2BgDXAneYWZrgbMf73F03RxURuYB0wZSISAScrnvlHBkVSkREzhcFvYhIxCnoRUQiTkEvIhJxc+5krJl1AS+ew1s0ASfOuNTcF5XtAG3LXBWVbYnKdsC5bctKd2+eacacC/pzZWadpzrzPJ9EZTtA2zJXRWVborIdcP62RU03IiIRp6AXEYm4KAb93cWuwCyJynaAtmWuisq2RGU74DxtS+Ta6EVEZLIoHtGLiEgeBb2ISMRFJujNbLOZ7TKzPWZ2e7Hrcy7MbJ+ZPW1m281sXo3wZmZfNLPjZvarvLJGM3vIzHaH04Zi1rFQp9iWj5nZofC72W5mv17MOhbCzJab2cNmttPMdpjZh8Lyefe9nGZb5uP3UmlmvzSzJ8Nt+cuwfLWZ/SLMsq+Gw8Of22dFoY2+kBuYzydmtg/ocPd5dxGImV0LDAFfGr8pvJl9Cuhx90+GO+EGd//jYtazEKfYlo8BQ+7+V8Ws29kwsxagxd0fN7MFwGPA24H3Ms++l9Nsy7uZf9+LATXuPmRmZcAjwIeADwPfcPd7zOxzwJPufk63ko3KEf3EDczdPQWM38BcLjB3/wnBPQny3Qj8c/j8nwn+Yc55p9iWecfdj7j74+HzQeAZgvs8z7vv5TTbMu94YCh8WRY+HHgj8PWwfFa+l6gE/Uw3MJ+XX37Ige+Z2WPhjdPnuyXufiR8fhRYUszKzILbzOypsGlnzjd35DOzVcBVwC+Y59/LlG2Befi9mFnczLYDx4GHCG632ufumXCRWcmyqAR91LzW3V8G3AB8IGxCiAQP2grnc3vhZ4G1wEbgCPDXRa3NWTCzWuA+4A/dfSB/3nz7XmbYlnn5vbh71t03EtyL+xrg4vPxOVEJ+kJuYD5vuPuhcHoc+CbB/wDz2bGwbXW8jfV4kevzkrn7sfAfZw74B+bJdxO2Ad8H/Ju7fyMsnpffy0zbMl+/l3Hu3gc8DLwKqDez8du8zkqWRSXoC7mB+bxgZjXhSSbMrAa4DvjV6dea8/JvHn8L8O0i1uWcjAdj6B3Mg+8mPOn3BeAZd78zb9a8+15OtS3z9HtpNrP68HkVQWeSZwgC/53hYrPyvUSi1w1A2J3qbzl5A/NPFLdGL42ZrSE4iofg5u1fnk/bYmZfAV5PMNzqMeCjwLeAe4EVBENQv3s+3CT+FNvyeoLmAQf2Ab+f1849J5nZa4GfAk8DubD4TwnatufV93KabbmZ+fe9XElwsjVOcNB9r7vfEWbAPUAj8ATwHndPntNnRSXoRURkZlFpuhERkVNQ0IuIRJyCXkQk4hT0IiIRp6AXEYk4Bb2ISMQp6EVEIu7/AxzewuEud6MdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['mae'])\n",
    "plt.plot(history.history['val_mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.006042855326086283, 0.0603807158768177]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = model.evaluate(X_test_pad, y_test_pad, verbose=0)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model_2():\n",
    "    ### YOUR CODE HERE\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(layers.Masking(mask_value=-1000, input_shape=(29,6)))\n",
    "    model.add(LSTM(units=10, activation='tanh', return_sequences=True))\n",
    "    model.add(layers.Dense(40, activation=\"relu\"))\n",
    "    model.add(layers.Dense(20, activation=\"relu\"))\n",
    "    model.add(layers.Dense(1, activation=\"linear\"))\n",
    "\n",
    "    model.compile(loss='mse',\n",
    "                  metrics=['mae'],\n",
    "                  optimizer='rmsprop') # Use `rmsprop`\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "13/13 [==============================] - 4s 85ms/step - loss: 0.0133 - mae: 0.0977 - val_loss: 0.0064 - val_mae: 0.0652\n",
      "Epoch 2/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0055 - mae: 0.0609 - val_loss: 0.0053 - val_mae: 0.0578\n",
      "Epoch 3/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0047 - mae: 0.0564 - val_loss: 0.0048 - val_mae: 0.0584\n",
      "Epoch 4/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0044 - mae: 0.0555 - val_loss: 0.0052 - val_mae: 0.0548\n",
      "Epoch 5/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0042 - mae: 0.0537 - val_loss: 0.0048 - val_mae: 0.0532\n",
      "Epoch 6/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0041 - mae: 0.0526 - val_loss: 0.0044 - val_mae: 0.0523\n",
      "Epoch 7/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0041 - mae: 0.0528 - val_loss: 0.0044 - val_mae: 0.0510\n",
      "Epoch 8/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0040 - mae: 0.0516 - val_loss: 0.0043 - val_mae: 0.0546\n",
      "Epoch 9/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0039 - mae: 0.0513 - val_loss: 0.0043 - val_mae: 0.0571\n",
      "Epoch 10/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0039 - mae: 0.0516 - val_loss: 0.0042 - val_mae: 0.0544\n",
      "Epoch 11/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0509 - val_loss: 0.0049 - val_mae: 0.0531\n",
      "Epoch 12/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0505 - val_loss: 0.0043 - val_mae: 0.0502\n",
      "Epoch 13/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0504 - val_loss: 0.0041 - val_mae: 0.0506\n",
      "Epoch 14/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0037 - mae: 0.0499 - val_loss: 0.0040 - val_mae: 0.0523\n",
      "Epoch 15/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0501 - val_loss: 0.0042 - val_mae: 0.0509\n",
      "Epoch 16/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0038 - mae: 0.0502 - val_loss: 0.0041 - val_mae: 0.0513\n",
      "Epoch 17/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0037 - mae: 0.0497 - val_loss: 0.0046 - val_mae: 0.0516\n",
      "Epoch 18/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0037 - mae: 0.0495 - val_loss: 0.0041 - val_mae: 0.0500\n",
      "Epoch 19/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0037 - mae: 0.0496 - val_loss: 0.0040 - val_mae: 0.0510\n",
      "Epoch 20/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0037 - mae: 0.0493 - val_loss: 0.0040 - val_mae: 0.0517\n",
      "Epoch 21/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0036 - mae: 0.0490 - val_loss: 0.0040 - val_mae: 0.0503\n",
      "Epoch 22/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0036 - mae: 0.0496 - val_loss: 0.0043 - val_mae: 0.0504\n",
      "Epoch 23/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0036 - mae: 0.0489 - val_loss: 0.0039 - val_mae: 0.0505\n",
      "Epoch 24/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0036 - mae: 0.0488 - val_loss: 0.0039 - val_mae: 0.0527\n",
      "Epoch 25/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0036 - mae: 0.0490 - val_loss: 0.0040 - val_mae: 0.0501\n",
      "Epoch 26/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0036 - mae: 0.0487 - val_loss: 0.0042 - val_mae: 0.0501\n",
      "Epoch 27/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0036 - mae: 0.0487 - val_loss: 0.0042 - val_mae: 0.0495\n",
      "Epoch 28/1000\n",
      "13/13 [==============================] - 0s 13ms/step - loss: 0.0035 - mae: 0.0481 - val_loss: 0.0039 - val_mae: 0.0502\n",
      "Epoch 29/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0035 - mae: 0.0485 - val_loss: 0.0040 - val_mae: 0.0490\n",
      "Epoch 30/1000\n",
      "13/13 [==============================] - 0s 14ms/step - loss: 0.0035 - mae: 0.0481 - val_loss: 0.0039 - val_mae: 0.0493\n",
      "Epoch 31/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0036 - mae: 0.0486 - val_loss: 0.0041 - val_mae: 0.0510\n",
      "Epoch 32/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0035 - mae: 0.0482 - val_loss: 0.0040 - val_mae: 0.0489\n",
      "Epoch 33/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0034 - mae: 0.0473 - val_loss: 0.0039 - val_mae: 0.0490\n",
      "Epoch 34/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0035 - mae: 0.0479 - val_loss: 0.0040 - val_mae: 0.0509\n",
      "Epoch 35/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0035 - mae: 0.0481 - val_loss: 0.0039 - val_mae: 0.0508\n",
      "Epoch 36/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0034 - mae: 0.0476 - val_loss: 0.0044 - val_mae: 0.0511\n",
      "Epoch 37/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0034 - mae: 0.0470 - val_loss: 0.0038 - val_mae: 0.0498\n",
      "Epoch 38/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0034 - mae: 0.0476 - val_loss: 0.0041 - val_mae: 0.0499\n",
      "Epoch 39/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0034 - mae: 0.0473 - val_loss: 0.0037 - val_mae: 0.0505\n",
      "Epoch 40/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0467 - val_loss: 0.0039 - val_mae: 0.0536\n",
      "Epoch 41/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0034 - mae: 0.0474 - val_loss: 0.0039 - val_mae: 0.0490\n",
      "Epoch 42/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0468 - val_loss: 0.0039 - val_mae: 0.0496\n",
      "Epoch 43/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0034 - mae: 0.0472 - val_loss: 0.0038 - val_mae: 0.0488\n",
      "Epoch 44/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0466 - val_loss: 0.0038 - val_mae: 0.0483\n",
      "Epoch 45/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0034 - mae: 0.0472 - val_loss: 0.0041 - val_mae: 0.0506\n",
      "Epoch 46/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0469 - val_loss: 0.0039 - val_mae: 0.0493\n",
      "Epoch 47/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0034 - mae: 0.0473 - val_loss: 0.0037 - val_mae: 0.0484\n",
      "Epoch 48/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0467 - val_loss: 0.0037 - val_mae: 0.0492\n",
      "Epoch 49/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0469 - val_loss: 0.0038 - val_mae: 0.0522\n",
      "Epoch 50/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0466 - val_loss: 0.0037 - val_mae: 0.0504\n",
      "Epoch 51/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0466 - val_loss: 0.0040 - val_mae: 0.0485\n",
      "Epoch 52/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0467 - val_loss: 0.0038 - val_mae: 0.0480\n",
      "Epoch 53/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0463 - val_loss: 0.0037 - val_mae: 0.0497\n",
      "Epoch 54/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0033 - mae: 0.0465 - val_loss: 0.0038 - val_mae: 0.0523\n",
      "Epoch 55/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0467 - val_loss: 0.0037 - val_mae: 0.0493\n",
      "Epoch 56/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0033 - mae: 0.0469 - val_loss: 0.0037 - val_mae: 0.0494\n",
      "Epoch 57/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0462 - val_loss: 0.0040 - val_mae: 0.0487\n",
      "Epoch 58/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0459 - val_loss: 0.0037 - val_mae: 0.0487\n",
      "Epoch 59/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0465 - val_loss: 0.0038 - val_mae: 0.0493\n",
      "Epoch 60/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0461 - val_loss: 0.0039 - val_mae: 0.0487\n",
      "Epoch 61/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0465 - val_loss: 0.0039 - val_mae: 0.0501\n",
      "Epoch 62/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0458 - val_loss: 0.0042 - val_mae: 0.0497\n",
      "Epoch 63/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0463 - val_loss: 0.0044 - val_mae: 0.0513\n",
      "Epoch 64/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0455 - val_loss: 0.0036 - val_mae: 0.0497\n",
      "Epoch 65/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0458 - val_loss: 0.0039 - val_mae: 0.0485\n",
      "Epoch 66/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0458 - val_loss: 0.0039 - val_mae: 0.0493\n",
      "Epoch 67/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0457 - val_loss: 0.0038 - val_mae: 0.0475\n",
      "Epoch 68/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0455 - val_loss: 0.0037 - val_mae: 0.0520\n",
      "Epoch 69/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0463 - val_loss: 0.0038 - val_mae: 0.0477\n",
      "Epoch 70/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0032 - mae: 0.0457 - val_loss: 0.0045 - val_mae: 0.0509\n",
      "Epoch 71/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0460 - val_loss: 0.0036 - val_mae: 0.0490\n",
      "Epoch 72/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0032 - mae: 0.0457 - val_loss: 0.0037 - val_mae: 0.0479\n",
      "Epoch 73/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0454 - val_loss: 0.0036 - val_mae: 0.0495\n",
      "Epoch 74/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0461 - val_loss: 0.0036 - val_mae: 0.0488\n",
      "Epoch 75/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0031 - mae: 0.0455 - val_loss: 0.0036 - val_mae: 0.0485\n",
      "Epoch 76/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0458 - val_loss: 0.0037 - val_mae: 0.0494\n",
      "Epoch 77/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0031 - mae: 0.0455 - val_loss: 0.0041 - val_mae: 0.0499\n",
      "Epoch 78/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0457 - val_loss: 0.0039 - val_mae: 0.0480\n",
      "Epoch 79/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0454 - val_loss: 0.0036 - val_mae: 0.0480\n",
      "Epoch 80/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0031 - mae: 0.0456 - val_loss: 0.0040 - val_mae: 0.0485\n",
      "Epoch 81/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0450 - val_loss: 0.0038 - val_mae: 0.0485\n",
      "Epoch 82/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0459 - val_loss: 0.0036 - val_mae: 0.0475\n",
      "Epoch 83/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0452 - val_loss: 0.0037 - val_mae: 0.0511\n",
      "Epoch 84/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0454 - val_loss: 0.0037 - val_mae: 0.0480\n",
      "Epoch 85/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0454 - val_loss: 0.0036 - val_mae: 0.0473\n",
      "Epoch 86/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0032 - mae: 0.0455 - val_loss: 0.0038 - val_mae: 0.0477\n",
      "Epoch 87/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0450 - val_loss: 0.0037 - val_mae: 0.0478\n",
      "Epoch 88/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0453 - val_loss: 0.0040 - val_mae: 0.0485\n",
      "Epoch 89/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0031 - mae: 0.0453 - val_loss: 0.0040 - val_mae: 0.0486\n",
      "Epoch 90/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0031 - mae: 0.0450 - val_loss: 0.0037 - val_mae: 0.0474\n",
      "Epoch 91/1000\n",
      "13/13 [==============================] - 0s 10ms/step - loss: 0.0031 - mae: 0.0454 - val_loss: 0.0038 - val_mae: 0.0477\n",
      "Epoch 92/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0449 - val_loss: 0.0039 - val_mae: 0.0485\n",
      "Epoch 93/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0451 - val_loss: 0.0039 - val_mae: 0.0483\n",
      "Epoch 94/1000\n",
      "13/13 [==============================] - 0s 11ms/step - loss: 0.0031 - mae: 0.0450 - val_loss: 0.0036 - val_mae: 0.0480\n",
      "Epoch 95/1000\n",
      "13/13 [==============================] - 0s 12ms/step - loss: 0.0032 - mae: 0.0456 - val_loss: 0.0036 - val_mae: 0.0493\n"
     ]
    }
   ],
   "source": [
    "model = init_model_2()\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=10, restore_best_weights=True)\n",
    "\n",
    "history = model.fit(X_train_pad, y_train_pad,\n",
    "            validation_split=0.2,\n",
    "            epochs=1000, \n",
    "            batch_size=64,\n",
    "            callbacks=[es], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.005420436151325703, 0.056760404258966446]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test_pad, y_test_pad, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}