QLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "button": false,
    "new_sheet": false,
    "run_control": {
     "read_only": false
    }
   },
   "outputs": [],
   "source": [
    "from mlagents.envs.environment import UnityEnvironment, BrainParameters\n",
    "import numpy as np\n",
    "from QLearningAgentAndy import *\n",
    "from function_approximators import *\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "button": false,
    "new_sheet": false,
    "run_control": {
     "read_only": false
    }
   },
   "outputs": [],
   "source": [
    "env = UnityEnvironment(file_name = \"environment-binaries/PushBlock-no-stack-small.app\", timeout_wait=5)\n",
    "# env = UnityEnvironment(file_name=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.step()\n",
    "env.reset()\n",
    "for name, brain in env.brains.items():\n",
    "    print(f\"Brain name: {name}\")\n",
    "    parameters = env.brains[name]\n",
    "    print(\"State space size:\\t\", parameters.vector_observation_space_size)\n",
    "    print(\"Action space size:\\t\", parameters.vector_action_space_size)\n",
    "    print(\"Action space type:\\t\", parameters.vector_action_space_type)\n",
    "\n",
    "state_space_size = parameters.vector_observation_space_size\n",
    "action_space_size = parameters.vector_observation_space_size\n",
    "action_space_type = parameters.vector_action_space_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def featureExtractor(state,action):\n",
    "    features = []\n",
    "    for i in range(len(state)):\n",
    "        features.append((i,state[i]))\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actions = (0,1,2,3,4,5,6)\n",
    "agent = QLearningAgent(actions, 1,featureExtractor,0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_episodes = 100\n",
    "max_timestep = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "info = env.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is merely a test to see how the agent reacts when prompted with actions\n",
    "#for i in range(100):\n",
    "#    info = env.step(0)[\"PushBlock\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check out observation and state size\n",
    "print(len((info[\"PushBlock\"].vector_observations)[0]))\n",
    "print(((info[\"PushBlock\"].vector_observations)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rewards_all_episodes = []\n",
    "for _ in tqdm.tqdm(range(max_episodes)):\n",
    "    rewards_current_episode = 0\n",
    "    info = env.step()[\"PushBlock\"]\n",
    "    for _ in range(max_timestep):\n",
    "        reward = info.rewards[0]\n",
    "        rewards_current_episode += reward\n",
    "        state = (info.vector_observations)[0]\n",
    "        action = agent.get_action(state)\n",
    "        info = env.step(action)[\"PushBlock\"]\n",
    "        nextState = info.vector_observations[0]\n",
    "        agent.give_feedback(state,action,reward,nextState)\n",
    "    rewards_all_episodes.append(rewards_current_episode)\n",
    "    env.reset(train_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "info = env.step()[\"PushBlock\"]\n",
    "print(info.vector_observations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Teardown Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python3"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "nteract": {
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}