Training_ANN_cls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from keras import *\n",
    "from keras.callbacks import *\n",
    "import os\n",
    "from sklearn.preprocessing import MinMaxScaler, RobustScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, classification_report\n",
    "from commons import mean_absolute_percentage_error\n",
    "from keras.layers import *\n",
    "from sklearn.pipeline import Pipeline\n",
    "from keras.utils import to_categorical\n",
    "from tensorflow.keras.models import load_model\n",
    "from keras.optimizers import *\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataset\n",
    "dataframe = pd.read_csv(\"pca_75_clas.csv\", sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "      <th>priceUSD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.074162</td>\n",
       "      <td>0.015329</td>\n",
       "      <td>-0.048046</td>\n",
       "      <td>0.042709</td>\n",
       "      <td>0.007321</td>\n",
       "      <td>-0.014251</td>\n",
       "      <td>0.001355</td>\n",
       "      <td>-0.044263</td>\n",
       "      <td>-0.014403</td>\n",
       "      <td>-0.036199</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017701</td>\n",
       "      <td>-0.020600</td>\n",
       "      <td>-0.021125</td>\n",
       "      <td>-0.001148</td>\n",
       "      <td>-0.004502</td>\n",
       "      <td>-0.012360</td>\n",
       "      <td>-0.032049</td>\n",
       "      <td>0.007081</td>\n",
       "      <td>0.006557</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.094841</td>\n",
       "      <td>0.072671</td>\n",
       "      <td>-0.077840</td>\n",
       "      <td>-0.014523</td>\n",
       "      <td>0.027039</td>\n",
       "      <td>-0.053013</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>-0.009060</td>\n",
       "      <td>0.047423</td>\n",
       "      <td>-0.009912</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.047544</td>\n",
       "      <td>0.013065</td>\n",
       "      <td>0.065670</td>\n",
       "      <td>0.006482</td>\n",
       "      <td>0.020321</td>\n",
       "      <td>0.007130</td>\n",
       "      <td>0.016320</td>\n",
       "      <td>0.013705</td>\n",
       "      <td>-0.042491</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.064880</td>\n",
       "      <td>0.028643</td>\n",
       "      <td>-0.038454</td>\n",
       "      <td>0.019065</td>\n",
       "      <td>0.028725</td>\n",
       "      <td>-0.014173</td>\n",
       "      <td>-0.002313</td>\n",
       "      <td>-0.031474</td>\n",
       "      <td>-0.009467</td>\n",
       "      <td>-0.034115</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020285</td>\n",
       "      <td>0.006481</td>\n",
       "      <td>-0.012896</td>\n",
       "      <td>0.008115</td>\n",
       "      <td>-0.022120</td>\n",
       "      <td>-0.021993</td>\n",
       "      <td>0.012241</td>\n",
       "      <td>0.021045</td>\n",
       "      <td>-0.033730</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  0.074162  0.015329 -0.048046  0.042709  0.007321 -0.014251  0.001355   \n",
       "1  0.094841  0.072671 -0.077840 -0.014523  0.027039 -0.053013  0.056817   \n",
       "2  0.064880  0.028643 -0.038454  0.019065  0.028725 -0.014173 -0.002313   \n",
       "\n",
       "          7         8         9  ...        41        42        43        44  \\\n",
       "0 -0.044263 -0.014403 -0.036199  ...  0.017701 -0.020600 -0.021125 -0.001148   \n",
       "1 -0.009060  0.047423 -0.009912  ... -0.047544  0.013065  0.065670  0.006482   \n",
       "2 -0.031474 -0.009467 -0.034115  ...  0.020285  0.006481 -0.012896  0.008115   \n",
       "\n",
       "         45        46        47        48        49  priceUSD  \n",
       "0 -0.004502 -0.012360 -0.032049  0.007081  0.006557         1  \n",
       "1  0.020321  0.007130  0.016320  0.013705 -0.042491         1  \n",
       "2 -0.022120 -0.021993  0.012241  0.021045 -0.033730         1  \n",
       "\n",
       "[3 rows x 51 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(735, 51)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "length=dataframe.shape[1]-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split into input (X) and output (Y) variables\n",
    "X = dataframe.iloc[:,0:length]\n",
    "y = dataframe['priceUSD']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>40</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.074162</td>\n",
       "      <td>0.015329</td>\n",
       "      <td>-0.048046</td>\n",
       "      <td>0.042709</td>\n",
       "      <td>0.007321</td>\n",
       "      <td>-0.014251</td>\n",
       "      <td>0.001355</td>\n",
       "      <td>-0.044263</td>\n",
       "      <td>-0.014403</td>\n",
       "      <td>-0.036199</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.004087</td>\n",
       "      <td>0.017701</td>\n",
       "      <td>-0.020600</td>\n",
       "      <td>-0.021125</td>\n",
       "      <td>-0.001148</td>\n",
       "      <td>-0.004502</td>\n",
       "      <td>-0.012360</td>\n",
       "      <td>-0.032049</td>\n",
       "      <td>0.007081</td>\n",
       "      <td>0.006557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.094841</td>\n",
       "      <td>0.072671</td>\n",
       "      <td>-0.077840</td>\n",
       "      <td>-0.014523</td>\n",
       "      <td>0.027039</td>\n",
       "      <td>-0.053013</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>-0.009060</td>\n",
       "      <td>0.047423</td>\n",
       "      <td>-0.009912</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003421</td>\n",
       "      <td>-0.047544</td>\n",
       "      <td>0.013065</td>\n",
       "      <td>0.065670</td>\n",
       "      <td>0.006482</td>\n",
       "      <td>0.020321</td>\n",
       "      <td>0.007130</td>\n",
       "      <td>0.016320</td>\n",
       "      <td>0.013705</td>\n",
       "      <td>-0.042491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.064880</td>\n",
       "      <td>0.028643</td>\n",
       "      <td>-0.038454</td>\n",
       "      <td>0.019065</td>\n",
       "      <td>0.028725</td>\n",
       "      <td>-0.014173</td>\n",
       "      <td>-0.002313</td>\n",
       "      <td>-0.031474</td>\n",
       "      <td>-0.009467</td>\n",
       "      <td>-0.034115</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014521</td>\n",
       "      <td>0.020285</td>\n",
       "      <td>0.006481</td>\n",
       "      <td>-0.012896</td>\n",
       "      <td>0.008115</td>\n",
       "      <td>-0.022120</td>\n",
       "      <td>-0.021993</td>\n",
       "      <td>0.012241</td>\n",
       "      <td>0.021045</td>\n",
       "      <td>-0.033730</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  0.074162  0.015329 -0.048046  0.042709  0.007321 -0.014251  0.001355   \n",
       "1  0.094841  0.072671 -0.077840 -0.014523  0.027039 -0.053013  0.056817   \n",
       "2  0.064880  0.028643 -0.038454  0.019065  0.028725 -0.014173 -0.002313   \n",
       "\n",
       "          7         8         9  ...        40        41        42        43  \\\n",
       "0 -0.044263 -0.014403 -0.036199  ... -0.004087  0.017701 -0.020600 -0.021125   \n",
       "1 -0.009060  0.047423 -0.009912  ...  0.003421 -0.047544  0.013065  0.065670   \n",
       "2 -0.031474 -0.009467 -0.034115  ...  0.014521  0.020285  0.006481 -0.012896   \n",
       "\n",
       "         44        45        46        47        48        49  \n",
       "0 -0.001148 -0.004502 -0.012360 -0.032049  0.007081  0.006557  \n",
       "1  0.006482  0.020321  0.007130  0.016320  0.013705 -0.042491  \n",
       "2  0.008115 -0.022120 -0.021993  0.012241  0.021045 -0.033730  \n",
       "\n",
       "[3 rows x 50 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=np.ravel(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1,\n",
       "       1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,\n",
       "       1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1,\n",
       "       1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0,\n",
       "       1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0,\n",
       "       0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0,\n",
       "       1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1,\n",
       "       0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0,\n",
       "       0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0,\n",
       "       1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0,\n",
       "       0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1,\n",
       "       1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0,\n",
       "       0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0,\n",
       "       0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
       "       1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0,\n",
       "       1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0,\n",
       "       0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0,\n",
       "       1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1,\n",
       "       1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0,\n",
       "       1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1,\n",
       "       0, 1, 1, 0, 1, 1, 0, 1, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "shape=X.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=0.2, train_size=0.8, shuffle=False, random_state=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimators=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimators.append(['robust',RobustScaler()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimators.append(['mixmax',MinMaxScaler()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale=Pipeline(estimators,verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Pipeline] ............ (step 1 of 2) Processing robust, total=   0.0s\n",
      "[Pipeline] ............ (step 2 of 2) Processing mixmax, total=   0.0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('robust',\n",
       "                 RobustScaler(copy=True, quantile_range=(25.0, 75.0),\n",
       "                              with_centering=True, with_scaling=True)),\n",
       "                ['mixmax', MinMaxScaler(copy=True, feature_range=(0, 1))]],\n",
       "         verbose=True)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scale.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=scale.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test=scale.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-2\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define neural network model\n",
    "def sequential_model(initializer='normal', activation='relu', neurons=300, NUM_FEATURES=shape):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(400, input_shape=(NUM_FEATURES,), kernel_initializer=initializer, activation=activation))\n",
    "    model.add(Dense(500, activation=activation))\n",
    "    model.add(Dense(100, activation=activation))\n",
    "    model.add(Dense(2, activation='softmax', kernel_initializer=initializer))\n",
    "    # Compile model\n",
    "    adam=Adam(lr=lr_schedule(0),amsgrad=True)\n",
    "    #sgd=keras.optimizers.SGD(learning_rate=0.08, momentum=0.9, nesterov=False)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcp_save = ModelCheckpoint('trained_models/ANN_cls_interval3_pca.hdf5', save_best_only=True, monitor='val_loss', mode='max')\n",
    "earlyStopping = EarlyStopping(monitor='val_loss', patience=100,verbose=1, mode='max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier=KerasClassifier(build_fn=sequential_model,batch_size=32, epochs=1000,validation_split=0.1,validation_freq=1, shuffle=True,use_multiprocessing=True, callbacks=[mcp_save,earlyStopping])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.01\n",
      "Train on 529 samples, validate on 59 samples\n",
      "Epoch 1/1000\n",
      "529/529 [==============================] - 0s 514us/step - loss: 0.7168 - accuracy: 0.5236 - val_loss: 0.6990 - val_accuracy: 0.4576\n",
      "Epoch 2/1000\n",
      "529/529 [==============================] - 0s 227us/step - loss: 0.6927 - accuracy: 0.5369 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 3/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6921 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 4/1000\n",
      "529/529 [==============================] - 0s 220us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 5/1000\n",
      "529/529 [==============================] - 0s 238us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 6/1000\n",
      "529/529 [==============================] - 0s 239us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.6994 - val_accuracy: 0.4576\n",
      "Epoch 7/1000\n",
      "529/529 [==============================] - 0s 240us/step - loss: 0.6908 - accuracy: 0.5331 - val_loss: 0.7034 - val_accuracy: 0.4576\n",
      "Epoch 8/1000\n",
      "529/529 [==============================] - 0s 243us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7024 - val_accuracy: 0.4576\n",
      "Epoch 9/1000\n",
      "529/529 [==============================] - 0s 232us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 10/1000\n",
      "529/529 [==============================] - 0s 222us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7005 - val_accuracy: 0.4576\n",
      "Epoch 11/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 12/1000\n",
      "529/529 [==============================] - 0s 296us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 13/1000\n",
      "529/529 [==============================] - 0s 240us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.6984 - val_accuracy: 0.4576\n",
      "Epoch 14/1000\n",
      "529/529 [==============================] - 0s 237us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6994 - val_accuracy: 0.4576\n",
      "Epoch 15/1000\n",
      "529/529 [==============================] - 0s 242us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7027 - val_accuracy: 0.4576\n",
      "Epoch 16/1000\n",
      "529/529 [==============================] - 0s 237us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7035 - val_accuracy: 0.4576\n",
      "Epoch 17/1000\n",
      "529/529 [==============================] - 0s 244us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 18/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7006 - val_accuracy: 0.4576\n",
      "Epoch 19/1000\n",
      "529/529 [==============================] - 0s 209us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 20/1000\n",
      "529/529 [==============================] - 0s 221us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 21/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6993 - val_accuracy: 0.4576\n",
      "Epoch 22/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6997 - val_accuracy: 0.4576\n",
      "Epoch 23/1000\n",
      "529/529 [==============================] - 0s 215us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 24/1000\n",
      "529/529 [==============================] - 0s 260us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 25/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 26/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 27/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7005 - val_accuracy: 0.4576\n",
      "Epoch 28/1000\n",
      "529/529 [==============================] - 0s 228us/step - loss: 0.6929 - accuracy: 0.5331 - val_loss: 0.7035 - val_accuracy: 0.4576\n",
      "Epoch 29/1000\n",
      "529/529 [==============================] - 0s 233us/step - loss: 0.6909 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 30/1000\n",
      "529/529 [==============================] - 0s 246us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6986 - val_accuracy: 0.4576\n",
      "Epoch 31/1000\n",
      "529/529 [==============================] - 0s 253us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7022 - val_accuracy: 0.4576\n",
      "Epoch 32/1000\n",
      "529/529 [==============================] - 0s 232us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7009 - val_accuracy: 0.4576\n",
      "Epoch 33/1000\n",
      "529/529 [==============================] - 0s 269us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 34/1000\n",
      "529/529 [==============================] - 0s 218us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 35/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 36/1000\n",
      "529/529 [==============================] - 0s 210us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 37/1000\n",
      "529/529 [==============================] - 0s 221us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 38/1000\n",
      "529/529 [==============================] - 0s 226us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7028 - val_accuracy: 0.4576\n",
      "Epoch 39/1000\n",
      "529/529 [==============================] - 0s 250us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6992 - val_accuracy: 0.4576\n",
      "Epoch 40/1000\n",
      "529/529 [==============================] - 0s 219us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 41/1000\n",
      "529/529 [==============================] - 0s 241us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 42/1000\n",
      "529/529 [==============================] - 0s 227us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 43/1000\n",
      "529/529 [==============================] - 0s 212us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 44/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7015 - val_accuracy: 0.4576\n",
      "Epoch 45/1000\n",
      "529/529 [==============================] - 0s 240us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7025 - val_accuracy: 0.4576\n",
      "Epoch 46/1000\n",
      "529/529 [==============================] - 0s 217us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7005 - val_accuracy: 0.4576\n",
      "Epoch 47/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7008 - val_accuracy: 0.4576\n",
      "Epoch 48/1000\n",
      "529/529 [==============================] - 0s 215us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 49/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6995 - val_accuracy: 0.4576\n",
      "Epoch 50/1000\n",
      "529/529 [==============================] - 0s 215us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 51/1000\n",
      "529/529 [==============================] - 0s 239us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 52/1000\n",
      "529/529 [==============================] - 0s 244us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7033 - val_accuracy: 0.4576\n",
      "Epoch 53/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6992 - val_accuracy: 0.4576\n",
      "Epoch 54/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7026 - val_accuracy: 0.4576\n",
      "Epoch 55/1000\n",
      "529/529 [==============================] - 0s 258us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7011 - val_accuracy: 0.4576\n",
      "Epoch 56/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "529/529 [==============================] - 0s 250us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 57/1000\n",
      "529/529 [==============================] - 0s 213us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7026 - val_accuracy: 0.4576\n",
      "Epoch 58/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 59/1000\n",
      "529/529 [==============================] - 0s 213us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 60/1000\n",
      "529/529 [==============================] - 0s 209us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 61/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 62/1000\n",
      "529/529 [==============================] - 0s 234us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 63/1000\n",
      "529/529 [==============================] - 0s 205us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 64/1000\n",
      "529/529 [==============================] - 0s 205us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7010 - val_accuracy: 0.4576\n",
      "Epoch 65/1000\n",
      "529/529 [==============================] - 0s 211us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 66/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7041 - val_accuracy: 0.4576\n",
      "Epoch 67/1000\n",
      "529/529 [==============================] - 0s 206us/step - loss: 0.6926 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 68/1000\n",
      "529/529 [==============================] - 0s 204us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 69/1000\n",
      "529/529 [==============================] - 0s 201us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 70/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7024 - val_accuracy: 0.4576\n",
      "Epoch 71/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6918 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 72/1000\n",
      "529/529 [==============================] - 0s 214us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 73/1000\n",
      "529/529 [==============================] - 0s 206us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7021 - val_accuracy: 0.4576\n",
      "Epoch 74/1000\n",
      "529/529 [==============================] - 0s 203us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7008 - val_accuracy: 0.4576\n",
      "Epoch 75/1000\n",
      "529/529 [==============================] - 0s 205us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7022 - val_accuracy: 0.4576\n",
      "Epoch 76/1000\n",
      "529/529 [==============================] - 0s 201us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 77/1000\n",
      "529/529 [==============================] - 0s 214us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 78/1000\n",
      "529/529 [==============================] - 0s 208us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7046 - val_accuracy: 0.4576\n",
      "Epoch 79/1000\n",
      "529/529 [==============================] - 0s 263us/step - loss: 0.6928 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 80/1000\n",
      "529/529 [==============================] - 0s 243us/step - loss: 0.6918 - accuracy: 0.5331 - val_loss: 0.7049 - val_accuracy: 0.4576\n",
      "Epoch 81/1000\n",
      "529/529 [==============================] - 0s 198us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 82/1000\n",
      "529/529 [==============================] - 0s 214us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 83/1000\n",
      "529/529 [==============================] - 0s 201us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 84/1000\n",
      "529/529 [==============================] - 0s 222us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7016 - val_accuracy: 0.4576\n",
      "Epoch 85/1000\n",
      "529/529 [==============================] - 0s 254us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 86/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7010 - val_accuracy: 0.4576\n",
      "Epoch 87/1000\n",
      "529/529 [==============================] - 0s 224us/step - loss: 0.6926 - accuracy: 0.5331 - val_loss: 0.6977 - val_accuracy: 0.4576\n",
      "Epoch 88/1000\n",
      "529/529 [==============================] - 0s 203us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6992 - val_accuracy: 0.4576\n",
      "Epoch 89/1000\n",
      "529/529 [==============================] - 0s 194us/step - loss: 0.6923 - accuracy: 0.5331 - val_loss: 0.7031 - val_accuracy: 0.4576\n",
      "Epoch 90/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7008 - val_accuracy: 0.4576\n",
      "Epoch 91/1000\n",
      "529/529 [==============================] - 0s 211us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 92/1000\n",
      "529/529 [==============================] - 0s 208us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.6983 - val_accuracy: 0.4576\n",
      "Epoch 93/1000\n",
      "529/529 [==============================] - 0s 215us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 94/1000\n",
      "529/529 [==============================] - 0s 207us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 95/1000\n",
      "529/529 [==============================] - 0s 258us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 96/1000\n",
      "529/529 [==============================] - 0s 246us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7009 - val_accuracy: 0.4576\n",
      "Epoch 97/1000\n",
      "529/529 [==============================] - 0s 219us/step - loss: 0.6923 - accuracy: 0.5331 - val_loss: 0.6974 - val_accuracy: 0.4576\n",
      "Epoch 98/1000\n",
      "529/529 [==============================] - 0s 205us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 99/1000\n",
      "529/529 [==============================] - 0s 207us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 100/1000\n",
      "529/529 [==============================] - 0s 233us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 101/1000\n",
      "529/529 [==============================] - 0s 253us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7020 - val_accuracy: 0.4576\n",
      "Epoch 102/1000\n",
      "529/529 [==============================] - 0s 252us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7016 - val_accuracy: 0.4576\n",
      "Epoch 103/1000\n",
      "529/529 [==============================] - 0s 210us/step - loss: 0.6920 - accuracy: 0.5331 - val_loss: 0.7030 - val_accuracy: 0.4576\n",
      "Epoch 104/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 105/1000\n",
      "529/529 [==============================] - 0s 262us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 106/1000\n",
      "529/529 [==============================] - 0s 225us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 107/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7021 - val_accuracy: 0.4576\n",
      "Epoch 108/1000\n",
      "529/529 [==============================] - 0s 239us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 109/1000\n",
      "529/529 [==============================] - 0s 218us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6993 - val_accuracy: 0.4576\n",
      "Epoch 110/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 111/1000\n",
      "529/529 [==============================] - 0s 243us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7020 - val_accuracy: 0.4576\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 112/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7027 - val_accuracy: 0.4576\n",
      "Epoch 113/1000\n",
      "529/529 [==============================] - 0s 248us/step - loss: 0.6922 - accuracy: 0.5331 - val_loss: 0.7051 - val_accuracy: 0.4576\n",
      "Epoch 114/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 115/1000\n",
      "529/529 [==============================] - 0s 242us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.6989 - val_accuracy: 0.4576\n",
      "Epoch 116/1000\n",
      "529/529 [==============================] - 0s 229us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 117/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 118/1000\n",
      "529/529 [==============================] - 0s 271us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 119/1000\n",
      "529/529 [==============================] - 0s 264us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7024 - val_accuracy: 0.4576\n",
      "Epoch 120/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7020 - val_accuracy: 0.4576\n",
      "Epoch 121/1000\n",
      "529/529 [==============================] - 0s 245us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 122/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 123/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7021 - val_accuracy: 0.4576\n",
      "Epoch 124/1000\n",
      "529/529 [==============================] - 0s 257us/step - loss: 0.6918 - accuracy: 0.5331 - val_loss: 0.6984 - val_accuracy: 0.4576\n",
      "Epoch 125/1000\n",
      "529/529 [==============================] - 0s 260us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7000 - val_accuracy: 0.4576\n",
      "Epoch 126/1000\n",
      "529/529 [==============================] - 0s 221us/step - loss: 0.6926 - accuracy: 0.5331 - val_loss: 0.7059 - val_accuracy: 0.4576\n",
      "Epoch 127/1000\n",
      "529/529 [==============================] - 0s 248us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7011 - val_accuracy: 0.4576\n",
      "Epoch 128/1000\n",
      "529/529 [==============================] - 0s 237us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 129/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7016 - val_accuracy: 0.4576\n",
      "Epoch 130/1000\n",
      "529/529 [==============================] - 0s 254us/step - loss: 0.6926 - accuracy: 0.5331 - val_loss: 0.7021 - val_accuracy: 0.4576\n",
      "Epoch 131/1000\n",
      "529/529 [==============================] - 0s 223us/step - loss: 0.6919 - accuracy: 0.5331 - val_loss: 0.6982 - val_accuracy: 0.4576\n",
      "Epoch 132/1000\n",
      "529/529 [==============================] - 0s 216us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 133/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 134/1000\n",
      "529/529 [==============================] - 0s 244us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 135/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7010 - val_accuracy: 0.4576\n",
      "Epoch 136/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 137/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7009 - val_accuracy: 0.4576\n",
      "Epoch 138/1000\n",
      "529/529 [==============================] - 0s 275us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 139/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 140/1000\n",
      "529/529 [==============================] - 0s 248us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 141/1000\n",
      "529/529 [==============================] - 0s 262us/step - loss: 0.6922 - accuracy: 0.5331 - val_loss: 0.6976 - val_accuracy: 0.4576\n",
      "Epoch 142/1000\n",
      "529/529 [==============================] - 0s 241us/step - loss: 0.6918 - accuracy: 0.5331 - val_loss: 0.7038 - val_accuracy: 0.4576\n",
      "Epoch 143/1000\n",
      "529/529 [==============================] - 0s 274us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 144/1000\n",
      "529/529 [==============================] - 0s 262us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 145/1000\n",
      "529/529 [==============================] - 0s 264us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 146/1000\n",
      "529/529 [==============================] - 0s 257us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7005 - val_accuracy: 0.4576\n",
      "Epoch 147/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6918 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 148/1000\n",
      "529/529 [==============================] - 0s 247us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 149/1000\n",
      "529/529 [==============================] - 0s 294us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 150/1000\n",
      "529/529 [==============================] - 0s 228us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7025 - val_accuracy: 0.4576\n",
      "Epoch 151/1000\n",
      "529/529 [==============================] - 0s 232us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7015 - val_accuracy: 0.4576\n",
      "Epoch 152/1000\n",
      "529/529 [==============================] - 0s 237us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7018 - val_accuracy: 0.4576\n",
      "Epoch 153/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 154/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 155/1000\n",
      "529/529 [==============================] - 0s 269us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7006 - val_accuracy: 0.4576\n",
      "Epoch 156/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 157/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 158/1000\n",
      "529/529 [==============================] - 0s 234us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7025 - val_accuracy: 0.4576\n",
      "Epoch 159/1000\n",
      "529/529 [==============================] - 0s 246us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 160/1000\n",
      "529/529 [==============================] - 0s 263us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7005 - val_accuracy: 0.4576\n",
      "Epoch 161/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 162/1000\n",
      "529/529 [==============================] - 0s 237us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7020 - val_accuracy: 0.4576\n",
      "Epoch 163/1000\n",
      "529/529 [==============================] - 0s 263us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 164/1000\n",
      "529/529 [==============================] - 0s 238us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7032 - val_accuracy: 0.4576\n",
      "Epoch 165/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 166/1000\n",
      "529/529 [==============================] - 0s 253us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6991 - val_accuracy: 0.4576\n",
      "Epoch 167/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "529/529 [==============================] - 0s 249us/step - loss: 0.6922 - accuracy: 0.5331 - val_loss: 0.7042 - val_accuracy: 0.4576\n",
      "Epoch 168/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 169/1000\n",
      "529/529 [==============================] - 0s 248us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7004 - val_accuracy: 0.4576\n",
      "Epoch 170/1000\n",
      "529/529 [==============================] - 0s 213us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7017 - val_accuracy: 0.4576\n",
      "Epoch 171/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7026 - val_accuracy: 0.4576\n",
      "Epoch 172/1000\n",
      "529/529 [==============================] - 0s 254us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7031 - val_accuracy: 0.4576\n",
      "Epoch 173/1000\n",
      "529/529 [==============================] - 0s 229us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.7000 - val_accuracy: 0.4576\n",
      "Epoch 174/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7022 - val_accuracy: 0.4576\n",
      "Epoch 175/1000\n",
      "529/529 [==============================] - 0s 244us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7026 - val_accuracy: 0.4576\n",
      "Epoch 176/1000\n",
      "529/529 [==============================] - 0s 255us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 177/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 178/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.6983 - val_accuracy: 0.4576\n",
      "Epoch 179/1000\n",
      "529/529 [==============================] - 0s 252us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.6981 - val_accuracy: 0.4576\n",
      "Epoch 180/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6917 - accuracy: 0.5331 - val_loss: 0.7025 - val_accuracy: 0.4576\n",
      "Epoch 181/1000\n",
      "529/529 [==============================] - 0s 230us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7013 - val_accuracy: 0.4576\n",
      "Epoch 182/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7016 - val_accuracy: 0.4576\n",
      "Epoch 183/1000\n",
      "529/529 [==============================] - 0s 257us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7003 - val_accuracy: 0.4576\n",
      "Epoch 184/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6919 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 185/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7022 - val_accuracy: 0.4576\n",
      "Epoch 186/1000\n",
      "529/529 [==============================] - 0s 276us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 187/1000\n",
      "529/529 [==============================] - 0s 241us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 188/1000\n",
      "529/529 [==============================] - 0s 263us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 189/1000\n",
      "529/529 [==============================] - 0s 262us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7015 - val_accuracy: 0.4576\n",
      "Epoch 190/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7050 - val_accuracy: 0.4576\n",
      "Epoch 191/1000\n",
      "529/529 [==============================] - 0s 260us/step - loss: 0.6921 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 192/1000\n",
      "529/529 [==============================] - 0s 269us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7032 - val_accuracy: 0.4576\n",
      "Epoch 193/1000\n",
      "529/529 [==============================] - 0s 254us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 194/1000\n",
      "529/529 [==============================] - 0s 252us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7002 - val_accuracy: 0.4576\n",
      "Epoch 195/1000\n",
      "529/529 [==============================] - 0s 223us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.6988 - val_accuracy: 0.4576\n",
      "Epoch 196/1000\n",
      "529/529 [==============================] - 0s 221us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7011 - val_accuracy: 0.4576\n",
      "Epoch 197/1000\n",
      "529/529 [==============================] - 0s 228us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6999 - val_accuracy: 0.4576\n",
      "Epoch 198/1000\n",
      "529/529 [==============================] - 0s 261us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7031 - val_accuracy: 0.4576\n",
      "Epoch 199/1000\n",
      "529/529 [==============================] - 0s 264us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 200/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6921 - accuracy: 0.5331 - val_loss: 0.7019 - val_accuracy: 0.4576\n",
      "Epoch 201/1000\n",
      "529/529 [==============================] - 0s 272us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6991 - val_accuracy: 0.4576\n",
      "Epoch 202/1000\n",
      "529/529 [==============================] - 0s 249us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7006 - val_accuracy: 0.4576\n",
      "Epoch 203/1000\n",
      "529/529 [==============================] - 0s 225us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7015 - val_accuracy: 0.4576\n",
      "Epoch 204/1000\n",
      "529/529 [==============================] - 0s 269us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.6996 - val_accuracy: 0.4576\n",
      "Epoch 205/1000\n",
      "529/529 [==============================] - 0s 236us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 206/1000\n",
      "529/529 [==============================] - 0s 242us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 207/1000\n",
      "529/529 [==============================] - 0s 267us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7007 - val_accuracy: 0.4576\n",
      "Epoch 208/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7001 - val_accuracy: 0.4576\n",
      "Epoch 209/1000\n",
      "529/529 [==============================] - 0s 226us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7016 - val_accuracy: 0.4576\n",
      "Epoch 210/1000\n",
      "529/529 [==============================] - 0s 231us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7029 - val_accuracy: 0.4576\n",
      "Epoch 211/1000\n",
      "529/529 [==============================] - 0s 261us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.7030 - val_accuracy: 0.4576\n",
      "Epoch 212/1000\n",
      "529/529 [==============================] - 0s 251us/step - loss: 0.6911 - accuracy: 0.5331 - val_loss: 0.7006 - val_accuracy: 0.4576\n",
      "Epoch 213/1000\n",
      "529/529 [==============================] - 0s 247us/step - loss: 0.6921 - accuracy: 0.5331 - val_loss: 0.6987 - val_accuracy: 0.4576\n",
      "Epoch 214/1000\n",
      "529/529 [==============================] - 0s 266us/step - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.7031 - val_accuracy: 0.4576\n",
      "Epoch 215/1000\n",
      "529/529 [==============================] - 0s 278us/step - loss: 0.6926 - accuracy: 0.5331 - val_loss: 0.6994 - val_accuracy: 0.4576\n",
      "Epoch 216/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6908 - accuracy: 0.5331 - val_loss: 0.7023 - val_accuracy: 0.4576\n",
      "Epoch 217/1000\n",
      "529/529 [==============================] - 0s 273us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7037 - val_accuracy: 0.4576\n",
      "Epoch 218/1000\n",
      "529/529 [==============================] - 0s 256us/step - loss: 0.6910 - accuracy: 0.5331 - val_loss: 0.7011 - val_accuracy: 0.4576\n",
      "Epoch 219/1000\n",
      "529/529 [==============================] - 0s 220us/step - loss: 0.6916 - accuracy: 0.5331 - val_loss: 0.6989 - val_accuracy: 0.4576\n",
      "Epoch 220/1000\n",
      "529/529 [==============================] - 0s 262us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7021 - val_accuracy: 0.4576\n",
      "Epoch 221/1000\n",
      "529/529 [==============================] - 0s 264us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7012 - val_accuracy: 0.4576\n",
      "Epoch 222/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "529/529 [==============================] - 0s 244us/step - loss: 0.6913 - accuracy: 0.5331 - val_loss: 0.7014 - val_accuracy: 0.4576\n",
      "Epoch 223/1000\n",
      "529/529 [==============================] - 0s 246us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.7015 - val_accuracy: 0.4576\n",
      "Epoch 224/1000\n",
      "529/529 [==============================] - 0s 246us/step - loss: 0.6909 - accuracy: 0.5331 - val_loss: 0.6998 - val_accuracy: 0.4576\n",
      "Epoch 225/1000\n",
      "529/529 [==============================] - 0s 233us/step - loss: 0.6914 - accuracy: 0.5331 - val_loss: 0.7020 - val_accuracy: 0.4576\n",
      "Epoch 226/1000\n",
      "529/529 [==============================] - 0s 219us/step - loss: 0.6912 - accuracy: 0.5331 - val_loss: 0.6997 - val_accuracy: 0.4576\n",
      "Epoch 00226: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7fe798267a90>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qyAUCY0FGwsM"
   },
   "outputs": [],
   "source": [
    "prediction_model = load_model('trained_models/ANN_cls_interval3_pca.hdf5',compile=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = prediction_model.predict_classes(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48299319727891155"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc=accuracy_score(y_test,y_pred)\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3146102477688323"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1=f1_score(y_test,y_pred,average='weighted')\n",
    "f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5254556779438762"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc=roc_auc_score(y_test,y_pred)\n",
    "auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob=[prediction_model.predict(X_test).max() for i in range(len(y_test))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    decrease       0.54      0.31      0.40       246\n",
      "    increase       0.52      0.74      0.61       248\n",
      "\n",
      "    accuracy                           0.53       494\n",
      "   macro avg       0.53      0.53      0.50       494\n",
      "weighted avg       0.53      0.53      0.50       494\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred,labels=[0,1], target_names=['decrease','increase']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=pd.DataFrame(zip(np.ravel(y_test),np.ravel(y_pred)),columns=['y_test','y_pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_test</th>\n",
       "      <th>y_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>471</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>494 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     y_test  y_pred\n",
       "0         0       0\n",
       "1         1       0\n",
       "2         1       0\n",
       "3         1       0\n",
       "4         0       1\n",
       "5         1       0\n",
       "6         1       0\n",
       "7         1       1\n",
       "8         1       1\n",
       "9         1       0\n",
       "10        0       0\n",
       "11        0       0\n",
       "12        1       0\n",
       "13        0       0\n",
       "14        0       0\n",
       "15        0       0\n",
       "16        1       1\n",
       "17        0       0\n",
       "18        1       0\n",
       "19        1       0\n",
       "20        1       0\n",
       "21        0       0\n",
       "22        0       0\n",
       "23        0       0\n",
       "24        1       0\n",
       "25        1       0\n",
       "26        1       0\n",
       "27        1       0\n",
       "28        0       0\n",
       "29        0       0\n",
       "..      ...     ...\n",
       "464       1       1\n",
       "465       0       1\n",
       "466       1       1\n",
       "467       1       1\n",
       "468       1       1\n",
       "469       0       1\n",
       "470       0       1\n",
       "471       0       1\n",
       "472       0       1\n",
       "473       0       1\n",
       "474       1       1\n",
       "475       0       1\n",
       "476       0       1\n",
       "477       0       0\n",
       "478       0       0\n",
       "479       0       0\n",
       "480       1       1\n",
       "481       1       1\n",
       "482       0       1\n",
       "483       1       1\n",
       "484       1       1\n",
       "485       0       1\n",
       "486       0       1\n",
       "487       0       1\n",
       "488       0       1\n",
       "489       1       1\n",
       "490       1       1\n",
       "491       0       1\n",
       "492       0       1\n",
       "493       0       1\n",
       "\n",
       "[494 rows x 2 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}