fixedPoles_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# import cv2\n",
    "# import csv\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import struct\n",
    "import h5py\n",
    "import scipy.io as sio\n",
    "# from scipy import ndimage\n",
    "from numpy import linalg as LA\n",
    "from IPython.display import display, Image\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "from six.moves import cPickle as pickle\n",
    "import tensorflow as tf\n",
    "\n",
    "# Config the matplotlib backend as plotting inline in IPython\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "# Load synthetic dataset\n",
    "\n",
    "X = scipy.io.loadmat('/Users/angelsrates/Documents/PhD/4th Semester/Project/poles_data2.mat')\n",
    "y = scipy.io.loadmat('/Users/angelsrates/Documents/PhD/4th Semester/Project/poles_y2.mat')\n",
    "data = X['data']\n",
    "data = np.squeeze(np.transpose(data))\n",
    "#data_noise = X['data_noise']\n",
    "#data_noise = np.squeeze(np.transpose(data_noise))\n",
    "#sys_par = np.squeeze(np.transpose(X['dic_par']))\n",
    "#sys_par = [np.append(np.array([-1]), sys_par[i]) for i in range(sys_par.shape[0])]\n",
    "y = y['label']\n",
    "y = np.squeeze(y - 1)\n",
    "n_classes = max(y) + 1\n",
    "#num_poles = np.squeeze(X['num_poles'])\n",
    "num_poles = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300, 50)\n"
     ]
    }
   ],
   "source": [
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sys_par = [[-1,1.39954943237774, -1], [-1,0.411382829503097, -1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.random.seed(4294967295)\n",
    "[N, T] = data.shape\n",
    "permutation = np.random.permutation(data.shape[0])\n",
    "data = [data[perm] for perm in permutation]\n",
    "y = [y[perm] for perm in permutation]\n",
    "X = data\n",
    "# If data with noise, change to:\n",
    "# X = data_noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data size (225, 50)\n",
      "Training Ground-Truth size (225,)\n",
      "Testing data size (75, 50)\n",
      "Testing Ground-Truth size (75,)\n"
     ]
    }
   ],
   "source": [
    "#Select training and testing (75% and 25%)\n",
    "thr = int(N*0.75)\n",
    "y = [int(i) for i in y]\n",
    "X_train = np.asarray(X[:thr])\n",
    "y_train = np.asarray(y[:thr])\n",
    "X_test = np.asarray(X[thr:])\n",
    "y_test = np.asarray(y[thr:])\n",
    "\n",
    "print('Training data size', X_train.shape)\n",
    "print('Training Ground-Truth size', y_train.shape)\n",
    "print('Testing data size', X_test.shape)\n",
    "print('Testing Ground-Truth size', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_batch_size(_train, step, batch_size):\n",
    "    # Function to fetch a \"batch_size\" amount of data from \"(X|y)_train\" data. \n",
    "    \n",
    "    shape = list(_train.shape)\n",
    "    #shape = list((batch_size, 1843200))\n",
    "    shape[0] = batch_size\n",
    "    #shape[1] = 1843200\n",
    "    batch_s = np.empty(shape)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        # Loop index\n",
    "        index = ((step-1)*batch_size + i) % len(_train)\n",
    "        batch_s[i] = _train[index]\n",
    "        #batch_s[i] = np.reshape(load_video(_train[index]), (1,1843200))\n",
    "\n",
    "    return batch_s\n",
    "\n",
    "def one_hot(y_):\n",
    "    # Function to encode output labels from number indexes \n",
    "    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]\n",
    "    \n",
    "    y_ = y_.reshape(len(y_))\n",
    "    n_values = np.max(y_) + 1\n",
    "    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Sum_888:0\", dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "from scipy import signal\n",
    "import control\n",
    "from scipy.signal import step2\n",
    "import math\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.0015\n",
    "batch_size = 1\n",
    "\n",
    "# Network Parameters\n",
    "n_input = T\n",
    "#dropout = 0.75 # Dropout, probability to keep units\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float64, [n_input])\n",
    "y = tf.placeholder(tf.float32, [1, n_classes])\n",
    "#labels = tf.placeholder(tf.int32, [1,1])\n",
    "\n",
    "def index_along_every_row(array, index):\n",
    "    N,_ = array.shape \n",
    "    return array[np.arange(N), index]\n",
    "\n",
    "def build_hankel_tensor(x, nr, nc, N, dim):\n",
    "    cidx = np.arange(0, nc, 1)\n",
    "    ridx = np.transpose(np.arange(1, nr+1, 1))\n",
    "    Hidx = np.transpose(np.tile(ridx, (nc,1))) + dim*np.tile(cidx, (nr,1))\n",
    "    Hidx = Hidx - 1\n",
    "    arr = tf.reshape(x[:], (1,N))\n",
    "    return tf.py_func(index_along_every_row, [arr, Hidx], [tf.float64])[0]\n",
    "\n",
    "def build_hankel(x, nr, nc, N, dim):\n",
    "    cidx = np.arange(0, nc, 1)\n",
    "    ridx = np.transpose(np.arange(1, nr+1, 1))\n",
    "    Hidx = np.transpose(np.tile(ridx, (nc,1))) + dim*np.tile(cidx, (nr,1))\n",
    "    Hidx = Hidx - 1\n",
    "    arr = x[:]\n",
    "    return arr[Hidx]\n",
    "\n",
    "# Create model\n",
    "def poles_net(x, sys_par, T, num_poles):\n",
    "    # Operate over single-channel trajectories\n",
    "    dim = 1\n",
    "    nr = int(T/(dim+1))*dim;\n",
    "    nc = T - int(T/(dim+1))+1;\n",
    "    H = build_hankel_tensor(x, nr, nc, T, dim)\n",
    "    [s, U, _] = tf.svd(H, full_matrices=True)\n",
    "    \n",
    "    # Rank calculation by singular values\n",
    "    val = tf.constant(1, tf.float64)\n",
    "    num_elements = tf.greater(s, val)\n",
    "    as_ints = tf.cast(num_elements, tf.int32)\n",
    "    r = tf.reduce_sum(as_ints)\n",
    "    print(r)\n",
    "    # Sampling rates at 0.3\n",
    "    W_col = []\n",
    "    var_list = []\n",
    "    for i in range(num_poles):\n",
    "        sys = control.TransferFunction([1, 0], sys_par[i], 0.3)\n",
    "        [y1, _] = control.matlab.impulse(sys, T=np.arange(T))\n",
    "        H1 = np.squeeze(build_hankel(y1[0,:T], nr, nc, T, dim))\n",
    "        [U1, s1, _] = np.linalg.svd(H1)\n",
    "        sp = tf.svd(tf.concat([U[:,:r], tf.stack(U1[:,:2])], 1), full_matrices=True, compute_uv=False)\n",
    "        var = tf.squeeze(tf.cast(tf.reduce_sum(tf.reshape(sp[:2], (1,2)), 1)/2, tf.float16))\n",
    "        var_list.append(sp)\n",
    "        magic_num = tf.constant(math.sqrt(2), tf.float16)\n",
    "        alpha_ind = tf.cond(tf.equal(var,magic_num), lambda: tf.constant(1), lambda: tf.constant(0))\n",
    "        W = tf.cond(tf.equal(alpha_ind, tf.constant(1)), lambda: tf.transpose(y1[:,:T]), lambda: tf.zeros([T, 1], dtype=tf.float64))\n",
    "        W_col.append(W)\n",
    "    v = tf.stack(var_list)\n",
    "    W = tf.reshape(tf.stack(W_col, axis=1), (T,num_poles))\n",
    "    coeff = tf.abs(tf.matrix_solve_ls(W, tf.reshape(x, (T,1)), l2_regularizer=0.0, fast=False, name=None))\n",
    "    coeff = tf.transpose(coeff)\n",
    "    out = tf.add(tf.matmul(tf.cast(coeff, tf.float32), weights['out']), biases['out'])\n",
    "    return [r, v, out]\n",
    "\n",
    "weights = {\n",
    "    'out': tf.Variable(tf.random_normal([num_poles, n_classes]))\n",
    "}\n",
    "    \n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([1, n_classes]))\n",
    "}\n",
    "\n",
    "[r, coeff, pred]= poles_net(x, sys_par, T, num_poles)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0).minimize(cost)\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_test = one_hot(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 3)\n",
      "Iter 1, Minibatch Loss= 0.350630, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 2, Minibatch Loss= 1.092495, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 3, Minibatch Loss= 0.532159, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 4, Minibatch Loss= 0.317341, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 5, Minibatch Loss= 0.220709, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 6, Minibatch Loss= 0.928536, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 7, Minibatch Loss= 0.437559, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 8, Minibatch Loss= 0.553559, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 9, Minibatch Loss= 1.355513, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 10, Minibatch Loss= 0.561859, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 11, Minibatch Loss= 0.331911, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 12, Minibatch Loss= 0.807425, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 13, Minibatch Loss= 1.028446, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 14, Minibatch Loss= 0.573566, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 15, Minibatch Loss= 0.729830, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 16, Minibatch Loss= 0.822875, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 17, Minibatch Loss= 0.437603, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 18, Minibatch Loss= 0.841129, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 19, Minibatch Loss= 0.471835, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 20, Minibatch Loss= 0.983757, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 21, Minibatch Loss= 0.495835, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 22, Minibatch Loss= 0.877923, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 23, Minibatch Loss= 0.490586, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 24, Minibatch Loss= 0.640221, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 25, Minibatch Loss= 0.352843, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 26, Minibatch Loss= 1.153351, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 27, Minibatch Loss= 0.413785, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 28, Minibatch Loss= 0.895169, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 29, Minibatch Loss= 0.458820, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 30, Minibatch Loss= 0.286006, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 31, Minibatch Loss= 0.807662, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 32, Minibatch Loss= 1.142372, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 33, Minibatch Loss= 0.542773, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 34, Minibatch Loss= 0.325336, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 35, Minibatch Loss= 0.901455, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 36, Minibatch Loss= 0.447806, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 37, Minibatch Loss= 1.066845, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 38, Minibatch Loss= 0.701378, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 39, Minibatch Loss= 0.389914, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 40, Minibatch Loss= 0.801694, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 41, Minibatch Loss= 0.453980, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 42, Minibatch Loss= 1.068539, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 43, Minibatch Loss= 0.530584, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 44, Minibatch Loss= 0.318505, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 45, Minibatch Loss= 0.811516, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 46, Minibatch Loss= 0.408691, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 47, Minibatch Loss= 0.259710, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 48, Minibatch Loss= 0.772163, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 49, Minibatch Loss= 0.380333, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 50, Minibatch Loss= 0.242280, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 51, Minibatch Loss= 1.534415, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 52, Minibatch Loss= 0.718732, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 53, Minibatch Loss= 0.390764, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 54, Minibatch Loss= 0.256289, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 55, Minibatch Loss= 0.866846, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 56, Minibatch Loss= 0.420473, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 57, Minibatch Loss= 0.262519, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 58, Minibatch Loss= 0.741202, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 59, Minibatch Loss= 0.367340, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 60, Minibatch Loss= 0.572854, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 61, Minibatch Loss= 1.532541, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 62, Minibatch Loss= 0.437486, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 63, Minibatch Loss= 0.700543, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 64, Minibatch Loss= 0.997152, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 65, Minibatch Loss= 0.537863, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 66, Minibatch Loss= 0.687529, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 67, Minibatch Loss= 0.376168, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 68, Minibatch Loss= 0.697995, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 69, Minibatch Loss= 0.448405, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 70, Minibatch Loss= 1.223802, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 71, Minibatch Loss= 0.439478, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 72, Minibatch Loss= 0.770942, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 73, Minibatch Loss= 0.476203, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 74, Minibatch Loss= 0.596116, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 75, Minibatch Loss= 1.131453, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 76, Minibatch Loss= 0.606501, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 77, Minibatch Loss= 0.729288, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 78, Minibatch Loss= 0.541123, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 79, Minibatch Loss= 0.937700, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 80, Minibatch Loss= 0.499688, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 81, Minibatch Loss= 0.664051, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 82, Minibatch Loss= 0.968324, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 83, Minibatch Loss= 0.637643, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 84, Minibatch Loss= 0.364441, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 85, Minibatch Loss= 0.245361, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 86, Minibatch Loss= 1.012528, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 87, Minibatch Loss= 0.480098, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 88, Minibatch Loss= 0.566854, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 89, Minibatch Loss= 1.202511, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 90, Minibatch Loss= 0.593502, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 91, Minibatch Loss= 0.655586, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 92, Minibatch Loss= 0.365175, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 93, Minibatch Loss= 1.031394, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 94, Minibatch Loss= 0.507601, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 95, Minibatch Loss= 0.929805, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 96, Minibatch Loss= 0.729408, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 97, Minibatch Loss= 0.401369, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 98, Minibatch Loss= 0.824110, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 99, Minibatch Loss= 0.458002, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 100, Minibatch Loss= 0.613473, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 101, Minibatch Loss= 1.123911, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 102, Minibatch Loss= 0.560354, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 103, Minibatch Loss= 0.676812, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 104, Minibatch Loss= 0.526756, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 105, Minibatch Loss= 0.566558, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 106, Minibatch Loss= 0.322150, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 107, Minibatch Loss= 1.246871, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 108, Minibatch Loss= 0.393601, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 109, Minibatch Loss= 0.259008, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 110, Minibatch Loss= 0.925740, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 111, Minibatch Loss= 0.384994, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 112, Minibatch Loss= 1.178877, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 113, Minibatch Loss= 0.573313, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 114, Minibatch Loss= 0.847026, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 115, Minibatch Loss= 0.519651, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 116, Minibatch Loss= 0.631305, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 117, Minibatch Loss= 0.995188, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 118, Minibatch Loss= 0.519372, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 119, Minibatch Loss= 0.802515, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 120, Minibatch Loss= 0.505113, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 121, Minibatch Loss= 0.612960, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 122, Minibatch Loss= 0.503616, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 123, Minibatch Loss= 0.297609, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 124, Minibatch Loss= 0.709856, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 125, Minibatch Loss= 0.360858, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 126, Minibatch Loss= 0.591626, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 127, Minibatch Loss= 0.317080, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 128, Minibatch Loss= 0.212381, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 129, Minibatch Loss= 0.766909, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 130, Minibatch Loss= 0.364641, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 131, Minibatch Loss= 1.730414, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 132, Minibatch Loss= 0.841510, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 133, Minibatch Loss= 0.700283, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 134, Minibatch Loss= 0.621098, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 135, Minibatch Loss= 0.845214, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 136, Minibatch Loss= 0.712721, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 137, Minibatch Loss= 0.620691, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 138, Minibatch Loss= 0.837456, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 139, Minibatch Loss= 0.535847, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 140, Minibatch Loss= 0.628698, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 141, Minibatch Loss= 0.979878, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 142, Minibatch Loss= 0.646630, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 143, Minibatch Loss= 0.672554, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 144, Minibatch Loss= 0.558538, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 145, Minibatch Loss= 0.567798, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 146, Minibatch Loss= 0.522888, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 147, Minibatch Loss= 1.171270, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 148, Minibatch Loss= 0.578298, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 149, Minibatch Loss= 0.649505, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 150, Minibatch Loss= 0.533989, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 151, Minibatch Loss= 0.556526, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 152, Minibatch Loss= 0.511300, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 153, Minibatch Loss= 0.514347, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 154, Minibatch Loss= 1.317093, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 155, Minibatch Loss= 0.447752, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 156, Minibatch Loss= 0.284098, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 157, Minibatch Loss= 0.204326, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 158, Minibatch Loss= 1.102578, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 159, Minibatch Loss= 1.025318, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 160, Minibatch Loss= 0.518653, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 161, Minibatch Loss= 0.797113, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 162, Minibatch Loss= 0.505248, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 163, Minibatch Loss= 0.948709, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 164, Minibatch Loss= 0.482956, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 165, Minibatch Loss= 0.670709, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 166, Minibatch Loss= 0.365613, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 167, Minibatch Loss= 1.116872, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 168, Minibatch Loss= 0.544328, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 169, Minibatch Loss= 0.611088, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 170, Minibatch Loss= 0.343410, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 171, Minibatch Loss= 0.697944, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 172, Minibatch Loss= 0.363228, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 173, Minibatch Loss= 1.310218, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 174, Minibatch Loss= 0.667564, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 175, Minibatch Loss= 0.589128, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 176, Minibatch Loss= 0.339262, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 177, Minibatch Loss= 0.231332, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 178, Minibatch Loss= 0.174054, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 179, Minibatch Loss= 1.313646, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 180, Minibatch Loss= 0.313633, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 181, Minibatch Loss= 1.099798, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 182, Minibatch Loss= 0.402145, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 183, Minibatch Loss= 0.957632, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 184, Minibatch Loss= 0.793866, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 185, Minibatch Loss= 0.427032, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 186, Minibatch Loss= 0.788435, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 187, Minibatch Loss= 0.902687, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 188, Minibatch Loss= 0.470419, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 189, Minibatch Loss= 0.750038, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 190, Minibatch Loss= 0.398550, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 191, Minibatch Loss= 1.037090, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 192, Minibatch Loss= 0.441399, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 193, Minibatch Loss= 0.280056, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 194, Minibatch Loss= 0.201715, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 195, Minibatch Loss= 1.186231, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 196, Minibatch Loss= 0.338816, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 197, Minibatch Loss= 0.230389, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 198, Minibatch Loss= 0.173195, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 199, Minibatch Loss= 0.138348, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 200, Minibatch Loss= 1.131494, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 201, Minibatch Loss= 0.490664, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 202, Minibatch Loss= 1.420785, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 203, Minibatch Loss= 0.691482, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 204, Minibatch Loss= 0.610271, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 205, Minibatch Loss= 0.349195, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 206, Minibatch Loss= 1.013828, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 207, Minibatch Loss= 0.813803, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 208, Minibatch Loss= 0.434712, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 209, Minibatch Loss= 0.278223, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 210, Minibatch Loss= 0.924887, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 211, Minibatch Loss= 1.002613, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 212, Minibatch Loss= 0.553053, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 213, Minibatch Loss= 0.329453, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 214, Minibatch Loss= 0.919331, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 215, Minibatch Loss= 0.910682, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 216, Minibatch Loss= 0.609322, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 217, Minibatch Loss= 0.352506, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 218, Minibatch Loss= 0.239408, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 219, Minibatch Loss= 1.050478, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 220, Minibatch Loss= 0.972155, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 221, Minibatch Loss= 0.535332, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 222, Minibatch Loss= 0.321670, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 223, Minibatch Loss= 0.950056, Training Accuracy= 0.00000\n",
      "(1, 3)\n",
      "Iter 224, Minibatch Loss= 0.467790, Training Accuracy= 1.00000\n",
      "(1, 3)\n",
      "Iter 225, Minibatch Loss= 0.606129, Training Accuracy= 1.00000\n",
      "Final Training Accuracy: 0.817777777778\n",
      "Optimization Finished!\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 0 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 1 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 2 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 3 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 4 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 5 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 6 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 7 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 8 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 9 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 10 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 11 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 12 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 13 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 14 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 15 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 16 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 17 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 18 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 19 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 20 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 21 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 22 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 23 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 24 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 25 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 26 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 27 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 28 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 29 [2, array([[ 1.03637025,  1.00156764,  0.9984299 ,  0.96225605],\n",
      "       [ 1.01166905,  1.00047129,  0.99952849,  0.98819317]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 30 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 31 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 32 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 33 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 34 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 35 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 36 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 37 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 38 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 39 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 40 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 41 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 42 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 43 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 44 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 45 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 46 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 47 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 48 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 49 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 50 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 51 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 52 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 53 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 54 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 55 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 56 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 57 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 58 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 59 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 60 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 61 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 62 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 63 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 64 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 65 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 66 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 67 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  1.  0.]]\n",
      "Trajectory: 68 [2, array([[ 1.03625253,  1.0014015 ,  0.99859653,  0.96238282],\n",
      "       [ 1.01173858,  1.00032944,  0.99967045,  0.98812198]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 69 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 70 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 71 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 72 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 1.  0.  0.]]\n",
      "Trajectory: 73 [2, array([[ 1.11792903,  1.08783311,  0.90366981,  0.86616089],\n",
      "       [ 1.07430566,  1.07253898,  0.92177012,  0.91971048]])]\n",
      "Testing Accuracy: 0.0\n",
      "[[ 0.  0.  1.]]\n",
      "Trajectory: 74 [4, array([[ 1.12330505,  1.08784953,  1.        ,  1.        ,  0.90365005,\n",
      "         0.85917738],\n",
      "       [ 1.07430838,  1.07320655,  1.        ,  1.        ,  0.92099278,\n",
      "         0.91970729]])]\n",
      "Testing Accuracy: 1.0\n",
      "Final Testing Accuracy: 0.413333333333\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "n_epochs = 1\n",
    "training_iters = X_train.shape[0]*n_epochs\n",
    "display_step = 1\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    train_acc = 0\n",
    "    while step * batch_size <= training_iters:\n",
    "        batch_x = np.squeeze(extract_batch_size(X_train,step,batch_size))\n",
    "        batch_y = extract_batch_size(one_hot(y_train),step,batch_size)\n",
    "        #batch_y = np.reshape(extract_batch_size(y_train,step,batch_size), (1,1))\n",
    "        print(batch_y.shape)\n",
    "        # Run optimization op (backprop)\n",
    "        sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch loss and accuracy\n",
    "            loss, acc = sess.run([cost, accuracy], feed_dict={x: batch_x,\n",
    "                                                              y: batch_y})\n",
    "            train_acc += acc \n",
    "            print(\"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc))\n",
    "        step += 1\n",
    "    print('Final Training Accuracy:', train_acc/(X_train.shape[0]*n_epochs))\n",
    "    print(\"Optimization Finished!\")\n",
    "    acc = 0\n",
    "    for i in range(X_test.shape[0]):\n",
    "        test = np.squeeze(X_test[i,:])\n",
    "        label = np.reshape(y_test[i], (1,n_classes))\n",
    "        #label = np.reshape(y_test[i], (1,1))\n",
    "        print(label)\n",
    "        print(\"Trajectory:\", i, \\\n",
    "            sess.run([r, coeff], feed_dict={x: test, y: label}))\n",
    "        print(\"Testing Accuracy:\", \\\n",
    "            sess.run(accuracy, feed_dict={x: test, y: label}))\n",
    "        acc += sess.run(accuracy, feed_dict={x: test, y: label})\n",
    "    print('Final Testing Accuracy:', acc/X_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}