q_agent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Author: Ariel Guerrero\n",
    "### ***Q-Learning Agent***\n",
    "\n",
    "\n",
    "Implementation of ***Q-Learning Agent*** using OpenAI Gym environment.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Q Learning Algorithm*\n",
    "\n",
    "***Q(s<sub>t</sub>, a<sub>t</sub>) = R<sub>t+1</sub> + gamma * max(Q(s<sub>t+1</sub>))***\n",
    "\n",
    "* s = state\n",
    "* a = action\n",
    "* R<sub>t+1</sub> = reward\n",
    "* gamma = discount factor\n",
    "* max(Q(s<sub>t+1</sub>) = max Q value ***for all possible actions*** in state s<sub>t+1</sub>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import gym\n",
    "import random\n",
    "import numpy as np\n",
    "import time\n",
    "from gym.envs.registration import register\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import pygame\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sometimes pygame doesnt close properly, ***run this cell to force it to close***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if pygame is running, quit it\n",
    "if pygame.get_init():\n",
    "    pygame.quit()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Registering A Frozen Lake Environment with no slippery tiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# frozen lake registration for non slippery lake\n",
    "try:\n",
    "    register(\n",
    "        id='FrozenLakeNoSlip-v0',\n",
    "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
    "        kwargs={'map_name': '4x4', 'is_slippery': False},\n",
    "        max_episode_steps=100,\n",
    "        reward_threshold=0.78,  # optimum = .8196\n",
    "    )\n",
    "except:\n",
    "    pass\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions to create a new environment and to display environment information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def new_env(env_name):\n",
    "    \"\"\"\n",
    "        description: Create a new environment.\n",
    "    \"\"\"\n",
    "    env = gym.make(env_name)\n",
    "    return env\n",
    "\n",
    "\n",
    "def env_attributes(env):\n",
    "    \"\"\"\n",
    "        description:    Prints Attributes of the environment\n",
    "        @param env:      Gym environment\n",
    "    \"\"\"\n",
    "    print(\"observation space: \", env.observation_space)\n",
    "    # number of actions\n",
    "    if type(\n",
    "            env.action_space) == gym.spaces.discrete.Discrete:\n",
    "        print(\"action space: \", env.action_space)\n",
    "    else:\n",
    "        print(\"action range: \",\n",
    "              env.action_space.low, env.action_space.high)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Definining the Agent and Q-Agent classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic Agent\n",
    "class Agent():\n",
    "    \"\"\"\n",
    "        description:    Agent with discrete or continuous action space\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, env):\n",
    "        # I know its discrete but putting this here for future reference\n",
    "        # is the agent discrete or continuous?\n",
    "        self.is_discrete = type(\n",
    "            env.action_space) == gym.spaces.discrete.Discrete\n",
    "\n",
    "        # if discrete, get the action size\n",
    "        if self.is_discrete:\n",
    "            self.action_size = env.action_space.n\n",
    "        else:\n",
    "            self.action_low = env.action_space.low\n",
    "            self.action_high = env.action_space.high\n",
    "            self.action_shape = env.action_space.shape\n",
    "\n",
    "        env_attributes(env)\n",
    "\n",
    "    def get_action(self, state):\n",
    "        # if discrete, get the action\n",
    "        if self.is_discrete:\n",
    "            action = random.randrange(self.action_size)\n",
    "        else:\n",
    "            action = np.random.uniform(\n",
    "                self.action_low,\n",
    "                self.action_high,\n",
    "                self.action_shape\n",
    "            )\n",
    "\n",
    "        return action\n",
    "\n",
    "\n",
    "# Q Learning Agent\n",
    "class QAgent(Agent):\n",
    "    def __init__(self, env, discount_rate=0.97, learning_rate=0.01, epsilon=.75):\n",
    "        \"\"\"\n",
    "            description:    Q Learning Agent\n",
    "            @param env:      Gym environment\n",
    "            @param discount_rate:   represents how much future values lose weight based on how far they are\n",
    "            @param learning_rate:   represents the rate at which an algorithm adjusts its estimates based on the new information \n",
    "            @param epsilon:    represents the probability of prioritizing an exploritory action over a policy action (explore values vs acting greedy)\n",
    "        \"\"\"\n",
    "        super().__init__(env)\n",
    "        # action size is already defined in the parent Agent class\n",
    "        # state size\n",
    "        self.state_size = env.observation_space.n\n",
    "        print(\"state size: \", self.state_size)\n",
    "        # discount rate\n",
    "        self.discount_rate = discount_rate\n",
    "        # learning rate\n",
    "        self.learning_rate = learning_rate\n",
    "        # epsilon\n",
    "        self.epsilon = epsilon\n",
    "        # building the model\n",
    "        self.build_model()\n",
    "\n",
    "    def build_model(self):\n",
    "        self.q_table = 1e-4 * \\\n",
    "            np.random.random([self.state_size, self.action_size])\n",
    "\n",
    "    # redefining the get_action method to use the Q table selecting values based on the corresponding state\n",
    "    def get_action(self, state):\n",
    "        q_state = self.q_table[state]\n",
    "        action_greedy = np.argmax(q_state)\n",
    "        action_explore = super().get_action(state)\n",
    "        # return a random action with probability epsilon else return the greedy action\n",
    "        return action_explore if random.random() < self.epsilon else action_greedy\n",
    "\n",
    "    def train(self, experience):\n",
    "        state, action, next_state, reward, done = experience\n",
    "\n",
    "        # get the next q state\n",
    "        q_next = self.q_table[next_state]\n",
    "        q_next = np.zeros([self.action_size]) if done else q_next\n",
    "        q_target = reward + self.discount_rate * np.max(q_next)\n",
    "        print(f\"target: {q_target}\")\n",
    "        print(f\"action: {action}\")\n",
    "        print(f\"q_next: {q_next}\")\n",
    "        print(f\"q_next[action]: {q_next[action]}\")\n",
    "        print(f\"state: {state}\")\n",
    "        print(f\"new_state: {next_state}\")    \n",
    "\n",
    "\n",
    "        print(f\"q_table: {self.q_table}\")\n",
    "        # calc distance from current q state to the target q state\n",
    "        print(f\"q_table[state][actions]: {self.q_table[state][action]}\")\n",
    "        q_update = q_target - self.q_table[state, action]\n",
    "        print(f\"q_update: {q_update}\")\n",
    "        self.q_table[state, action] += self.learning_rate * q_update\n",
    "        print(f\"q_table: {self.q_table}\")\n",
    "        # we need to reduce epsilon over time as we get more experience and the table is more optimal\n",
    "        if done:\n",
    "            # exponential decay to epsilon\n",
    "            self.epsilon = self.epsilon * 0.99\n",
    "        \n",
    "        # wait 2 seconds\n",
    "        time.sleep(4)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializing the Agent and Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space:  Discrete(16)\n",
      "action space:  Discrete(4)\n",
      "state size:  16\n"
     ]
    }
   ],
   "source": [
    "#  lake is slippery (hard)\n",
    "# env_name = 'FrozenLake-v1'\n",
    "#  lake is not slippery (easy)\n",
    "env_name = 'FrozenLakeNoSlip-v0'\n",
    "# create new environment\n",
    "env = new_env(env_name)\n",
    "#  new QAgent\n",
    "agent = QAgent(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Driver Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "target: 8.763686624751133e-05\n",
      "action: 1\n",
      "q_next: [8.97646178e-06 9.03472848e-05 5.96939136e-05 7.66099532e-05]\n",
      "q_next[action]: 9.034728479124879e-05\n",
      "state: 0\n",
      "new_state: 4\n",
      "q_table: [[3.88907177e-05 2.95443311e-05 3.09162971e-05 1.04648915e-05]\n",
      " [5.41589639e-06 6.56875818e-05 7.15945535e-06 2.53001402e-05]\n",
      " [7.12741330e-05 7.81181762e-05 3.04170225e-05 8.57554077e-06]\n",
      " [2.08127846e-05 4.32472924e-05 9.83342561e-05 1.22447720e-05]\n",
      " [8.97646178e-06 9.03472848e-05 5.96939136e-05 7.66099532e-05]\n",
      " [1.70629732e-05 1.63644470e-05 5.34573245e-05 9.94798507e-05]\n",
      " [5.52145438e-05 6.83857442e-05 9.17767473e-06 1.50099803e-05]\n",
      " [5.68236839e-05 2.79478341e-05 6.58232214e-05 2.23804455e-05]\n",
      " [9.00728209e-05 1.35005627e-05 8.39464914e-05 5.06668516e-05]\n",
      " [1.91060263e-05 9.72176101e-05 1.25837108e-05 5.62348179e-05]\n",
      " [2.84159225e-05 7.28000724e-05 8.46431348e-05 9.66354304e-05]\n",
      " [1.38454249e-05 8.63095651e-05 1.71780381e-05 3.33302087e-05]\n",
      " [4.70389806e-05 3.44168153e-05 5.88337038e-05 4.16891086e-06]\n",
      " [9.88727278e-05 8.95796899e-05 4.59088677e-05 4.19223015e-05]\n",
      " [6.15223721e-07 2.38996711e-05 1.78039411e-05 7.64030574e-05]\n",
      " [5.57613372e-05 7.52528274e-05 3.27156384e-05 4.43754806e-06]]\n",
      "q_table[state][actions]: 2.9544331094232103e-05\n",
      "q_update: 5.809253515327923e-05\n",
      "q_table: [[3.88907177e-05 3.01252564e-05 3.09162971e-05 1.04648915e-05]\n",
      " [5.41589639e-06 6.56875818e-05 7.15945535e-06 2.53001402e-05]\n",
      " [7.12741330e-05 7.81181762e-05 3.04170225e-05 8.57554077e-06]\n",
      " [2.08127846e-05 4.32472924e-05 9.83342561e-05 1.22447720e-05]\n",
      " [8.97646178e-06 9.03472848e-05 5.96939136e-05 7.66099532e-05]\n",
      " [1.70629732e-05 1.63644470e-05 5.34573245e-05 9.94798507e-05]\n",
      " [5.52145438e-05 6.83857442e-05 9.17767473e-06 1.50099803e-05]\n",
      " [5.68236839e-05 2.79478341e-05 6.58232214e-05 2.23804455e-05]\n",
      " [9.00728209e-05 1.35005627e-05 8.39464914e-05 5.06668516e-05]\n",
      " [1.91060263e-05 9.72176101e-05 1.25837108e-05 5.62348179e-05]\n",
      " [2.84159225e-05 7.28000724e-05 8.46431348e-05 9.66354304e-05]\n",
      " [1.38454249e-05 8.63095651e-05 1.71780381e-05 3.33302087e-05]\n",
      " [4.70389806e-05 3.44168153e-05 5.88337038e-05 4.16891086e-06]\n",
      " [9.88727278e-05 8.95796899e-05 4.59088677e-05 4.19223015e-05]\n",
      " [6.15223721e-07 2.38996711e-05 1.78039411e-05 7.64030574e-05]\n",
      " [5.57613372e-05 7.52528274e-05 3.27156384e-05 4.43754806e-06]]\n",
      "state: 4, action: 1\n",
      "episode: 1, total reward: 0.0, epsilon: 0.7424999999999999\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb Cell 15'\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb#ch0000014vscode-remote?line=15'>16</a>\u001b[0m \u001b[39mprint\u001b[39m(\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb#ch0000014vscode-remote?line=16'>17</a>\u001b[0m     \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mepisode: \u001b[39m\u001b[39m{\u001b[39;00mepisode\u001b[39m}\u001b[39;00m\u001b[39m, total reward: \u001b[39m\u001b[39m{\u001b[39;00mtotal_reward\u001b[39m}\u001b[39;00m\u001b[39m, epsilon: \u001b[39m\u001b[39m{\u001b[39;00magent\u001b[39m.\u001b[39mepsilon\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb#ch0000014vscode-remote?line=17'>18</a>\u001b[0m env\u001b[39m.\u001b[39mrender()\n\u001b[0;32m---> <a href='vscode-notebook-cell://wsl%2Bubuntu/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb#ch0000014vscode-remote?line=18'>19</a>\u001b[0m time\u001b[39m.\u001b[39;49msleep(\u001b[39m.05\u001b[39;49m)\n\u001b[1;32m     <a href='vscode-notebook-cell://wsl%2Bubuntu/mnt/c/Users/holai/code/ai-coding-practice/q_agent.ipynb#ch0000014vscode-remote?line=19'>20</a>\u001b[0m clear_output(wait\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# used to keep track of the cumulative reward\n",
    "total_reward = 0\n",
    "# train agent over a number of episodes\n",
    "for episode in range(100):\n",
    "    # fresh state\n",
    "    state = env.reset()\n",
    "    # agent is finished or failed\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        agent.train(experience=(state, action, next_state, reward, done))\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "        print(f\"state: {state}, action: {action}\")\n",
    "        print(\n",
    "            f\"episode: {episode}, total reward: {total_reward}, epsilon: {agent.epsilon}\")\n",
    "        # env.render()\n",
    "        time.sleep(.05)\n",
    "        clear_output(wait=True)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}