Double_DQN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Double DQN",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vincent1rookie/RL-Lab/blob/master/Double_DQN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qsFw495uM1mp",
        "colab_type": "text"
      },
      "source": [
        "# Double DQN\n",
        "\n",
        "## 1. Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FpyCE9IAN2qX",
        "colab_type": "text"
      },
      "source": [
        "## 2. Initialize the environment dependencies\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "The same as what we did in Policy gradient."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NtJfSCCkbvD_",
        "colab_type": "code",
        "outputId": "2d490b38-f9c9-4a0b-9b41-80a6dcb4ca24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "!pip install gym pyvirtualdisplay > /dev/null 2>&1\n",
        "!apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1\n",
        "!apt-get update > /dev/null 2>&1\n",
        "!apt-get install cmake > /dev/null 2>&1\n",
        "!pip install --upgrade setuptools 2>&1\n",
        "!pip install ez_setup > /dev/null 2>&1\n",
        "!pip install gym[atari] > /dev/null 2>&1\n",
        "!pip install box2d-py\n",
        "!pip install gym[Box_2D]"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: setuptools in /usr/local/lib/python3.6/dist-packages (41.0.1)\n",
            "Collecting box2d-py\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/06/bd/6cdc3fd994b0649dcf5d9bad85bd9e26172308bbe9a421bfc6fdbf5081a6/box2d_py-2.3.8-cp36-cp36m-manylinux1_x86_64.whl (448kB)\n",
            "\u001b[K     |████████████████████████████████| 450kB 3.4MB/s \n",
            "\u001b[?25hInstalling collected packages: box2d-py\n",
            "Successfully installed box2d-py-2.3.8\n",
            "Requirement already satisfied: gym[Box_2D] in /usr/local/lib/python3.6/dist-packages (0.10.11)\n",
            "\u001b[33m  WARNING: gym 0.10.11 does not provide the extra 'box_2d'\u001b[0m\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym[Box_2D]) (1.3.0)\n",
            "Requirement already satisfied: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym[Box_2D]) (1.3.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gym[Box_2D]) (1.12.0)\n",
            "Requirement already satisfied: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym[Box_2D]) (2.21.0)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym[Box_2D]) (1.16.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym[Box_2D]) (0.16.0)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[Box_2D]) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[Box_2D]) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[Box_2D]) (2019.3.9)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[Box_2D]) (2.8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeba0v8vclRU",
        "colab_type": "code",
        "outputId": "0f76fe2b-6a31-4d6b-e932-bfc05334abd3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "import gym\n",
        "from gym import logger as gymlogger\n",
        "from gym.wrappers import Monitor\n",
        "gymlogger.set_level(40) #error only\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import random\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import math\n",
        "import glob\n",
        "import io\n",
        "import base64\n",
        "import os\n",
        "from IPython.display import HTML\n",
        "from IPython import display as ipythondisplay\n",
        "\n",
        "from pyvirtualdisplay import Display\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Display cmd_param=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1001'] cmd=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1001'] oserror=None return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BkMk98MlbF8v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "\n",
        "\n",
        "class DoubleQNetworkAgent:\n",
        "\n",
        "    def __init__(self, name: str, n_action: int, n_obs: int, units_layer: tuple,\n",
        "                 learning_rate=1e-4, gamma=0.99, seed=1,\n",
        "                 epsilon_init=0, epsilon_increase=0.003, epsilon_max=0.95,\n",
        "                 buffer_size=50000, batch_size=32, target_change_step=200, min_buffer_size=10000,\n",
        "                 save_path=None, load_path=None):\n",
        "        \"\"\"\n",
        "        To initialize an agent based on double deep Q learning\n",
        "        :param name: name of model\n",
        "        :param n_action: dimension of action space\n",
        "        :param n_obs: dimension of state space\n",
        "        :param units_layer: number of hidden layers and numbers of units in each layer\n",
        "        :param learning_rate: learning rate of optimizer\n",
        "        :param gamma: reward discount\n",
        "        :param seed: random seed\n",
        "        :param epsilon_init: initial epsilon value for epsilon-greedy policy\n",
        "        :param epsilon_increase: increase step for epsilon after each certain learning step\n",
        "        :param epsilon_max: upper threshold for epsilon\n",
        "        :param buffer_size: refers maximum size of transitions in buffer\n",
        "        :param batch_size: transition used in each training step\n",
        "        :param target_change_step: length of period for each refresh of Target Q-Network\n",
        "        :param min_buffer_size: minimum transitions for training process to initiate\n",
        "        :param save_path: path to save model variables\n",
        "        :param load_path: path to reload model variables\n",
        "        \"\"\"\n",
        "        self.name = name\n",
        "        self.n_action = n_action\n",
        "        self.n_obs = n_obs\n",
        "        self.n_layer = len(units_layer)\n",
        "        self.gamma = gamma\n",
        "        # self.tau = tau\n",
        "        self.learning_rate = learning_rate\n",
        "        self.epsilon = epsilon_init\n",
        "        self.epsilon_increase = epsilon_increase\n",
        "        self.epsilon_max = epsilon_max\n",
        "        self.initializer = tf.contrib.layers.xavier_initializer(seed=seed)\n",
        "        self.units_layer = (self.n_obs,) + units_layer + (self.n_action,)\n",
        "        self.sess = tf.Session()\n",
        "        self.transition_count = 0\n",
        "        self.buffer_size = buffer_size\n",
        "        self.min_buffer_size = min_buffer_size\n",
        "        self.batch_size = batch_size\n",
        "        self.target_change_step = target_change_step\n",
        "        self.buffer_list = []\n",
        "        self.learn_step_count = 0\n",
        "        self.l_history = []\n",
        "\n",
        "        with tf.variable_scope(self.name + '_input', reuse=tf.AUTO_REUSE):\n",
        "            self.s = tf.placeholder(tf.float32, [None, self.n_obs], name='s')\n",
        "            self.s_next = tf.placeholder(tf.float32, [None, self.n_obs], name='s_next')\n",
        "            self.a = tf.placeholder(tf.int32, name='a')\n",
        "            self.r = tf.placeholder(tf.float32, name='r')\n",
        "            self.d = tf.placeholder(tf.float32, name='d')\n",
        "\n",
        "        with tf.variable_scope(self.name + '_main_params', reuse=tf.AUTO_REUSE):\n",
        "            self.W_main, self.b_main = [], []\n",
        "            for i in range(self.n_layer + 1):\n",
        "                self.W_main.append(tf.get_variable('W' + str(i),\n",
        "                                                   [self.units_layer[i], self.units_layer[i + 1]],\n",
        "                                                   initializer=self.initializer))\n",
        "                self.b_main.append(tf.get_variable('b' + str(i),\n",
        "                                                   [1, self.units_layer[i + 1]],\n",
        "                                                   initializer=self.initializer))\n",
        "\n",
        "        with tf.variable_scope(self.name + '_main_layers', reuse=tf.AUTO_REUSE):\n",
        "            self.layer_main = [self.s]\n",
        "            for i in range(self.n_layer + 1):\n",
        "                if i < self.n_layer:\n",
        "                    self.layer_main.append(\n",
        "                        tf.nn.relu(tf.add(tf.matmul(self.layer_main[i], self.W_main[i]), self.b_main[i])))\n",
        "                else:\n",
        "                    self.layer_main.append(\n",
        "                        tf.add(tf.matmul(self.layer_main[i], self.W_main[i]), self.b_main[i]))\n",
        "\n",
        "        with tf.variable_scope(self.name + '_target_params', reuse=tf.AUTO_REUSE):\n",
        "            self.W_target, self.b_target = [], []\n",
        "            for i in range(self.n_layer + 1):\n",
        "                self.W_target.append(tf.Variable(self.W_main[i].initialized_value(), name='W' + str(i)))\n",
        "                self.b_target.append(tf.Variable(self.b_main[i].initialized_value(), name='b' + str(i)))\n",
        "\n",
        "        with tf.variable_scope(self.name + '_target_layers', reuse=tf.AUTO_REUSE):\n",
        "            self.layer_target = [self.s_next]\n",
        "            for i in range(self.n_layer + 1):\n",
        "                if i < self.n_layer:\n",
        "                    self.layer_target.append(\n",
        "                        tf.nn.relu(tf.add(tf.matmul(self.layer_target[i], self.W_target[i]), self.b_target[i])))\n",
        "                else:\n",
        "                    self.layer_target.append(\n",
        "                        tf.add(tf.matmul(self.layer_target[i], self.W_target[i]), self.b_target[i]))\n",
        "\n",
        "        with tf.variable_scope(self.name + '_loss', reuse=tf.AUTO_REUSE):\n",
        "            # Choose best action based on main Q-Network\n",
        "            self.action_next_one_hot = tf.one_hot(tf.argmax(self.layer_main[-1], axis=1), self.n_action)\n",
        "            self.action_one_hot = tf.one_hot(self.a, self.n_action)\n",
        "            self.Q_real = tf.reduce_sum(self.layer_target[-1] * self.action_next_one_hot, axis=1) * self.gamma * (\n",
        "                        tf.ones_like(self.d) - self.d) + self.r\n",
        "            self.Q_eval = tf.reduce_sum(self.layer_main[-1] * self.action_one_hot, axis=1)\n",
        "            self.td_error = tf.square(self.Q_eval - self.Q_real)\n",
        "            self.loss = tf.reduce_mean(self.td_error)\n",
        "\n",
        "        with tf.variable_scope(self.name + '_op', reuse=tf.AUTO_REUSE):\n",
        "            self.op = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.loss)\n",
        "\n",
        "        # with tf.variable_scope(self.name + 'target_renew', reuse=tf.AUTO_REUSE):\n",
        "        #     for i in range(self.n_layer+1):\n",
        "        #         self.sess.run(self.W_target[i] += self.tau * (self.W_main[i] - self.W_target[i]))\n",
        "        #         self.sess.run(self.b_target[i] += self.tau * (self.b_main[i] - self.b_target[i]))\n",
        "\n",
        "        # Try to save and/or reload model\n",
        "        self.save_path = save_path\n",
        "        self.saver = tf.train.Saver()\n",
        "        if load_path is not None:\n",
        "            self.load_path = load_path\n",
        "            self.saver.restore(self.sess, self.load_path)\n",
        "        else:\n",
        "            self.sess.run(tf.global_variables_initializer())\n",
        "\n",
        "    def act(self, obs, test_mode=False):\n",
        "        \"\"\"\n",
        "        To make an action when received a new observation based on epsilon-greedy policy.\n",
        "\n",
        "        :param obs: observation\n",
        "        :param test_mode: if True, will not apply epsilon-greedy policy\n",
        "        :return: an int indicating corresponding action\n",
        "        \"\"\"\n",
        "        if np.random.uniform() < self.epsilon or test_mode:\n",
        "            return np.argmax(self.sess.run(self.layer_main[-1], feed_dict={self.s: obs.reshape([1, self.n_obs])}))\n",
        "        else:\n",
        "            return np.random.randint(0, self.n_action)\n",
        "\n",
        "    def record(self, state: np.ndarray, action: int, reward: float, state_next: np.ndarray, done: bool):\n",
        "        \"\"\"\n",
        "        To record a transition of (s,a,r,s_next,done)  into replay buffer\n",
        "        \"\"\"\n",
        "        transition = np.hstack([state.reshape([1, self.n_obs]), np.array([action, reward, done]).reshape([1, 3]),\n",
        "                                state_next.reshape([1, self.n_obs])])\n",
        "        if self.transition_count < self.buffer_size:\n",
        "            self.buffer_list.append(transition)\n",
        "        else:\n",
        "            index = self.transition_count % self.buffer_size\n",
        "            self.buffer_list[index] = transition\n",
        "        self.transition_count += 1\n",
        "\n",
        "    def learn(self):\n",
        "        \"\"\"\n",
        "        To train the model after each transition\n",
        "        \"\"\"\n",
        "        if len(self.buffer_list) <= self.min_buffer_size:\n",
        "            return\n",
        "        # Sample from buffer\n",
        "        sample = np.vstack(random.sample(self.buffer_list, self.batch_size))\n",
        "        s = sample[:, :self.n_obs]\n",
        "        a = sample[:, self.n_obs:self.n_obs + 1].flatten()\n",
        "        r = sample[:, self.n_obs + 1: self.n_obs + 2].flatten()\n",
        "        d = sample[:, self.n_obs + 2: self.n_obs + 3].flatten()\n",
        "        s_next = sample[:, self.n_obs + 3:]\n",
        "\n",
        "        # Train the main DQN and record loss\n",
        "        self.sess.run(self.op, feed_dict={self.s: s, self.a: a, self.r: r, self.s_next: s_next, self.d: d})\n",
        "\n",
        "        self.l_history.append(\n",
        "            self.sess.run(self.loss, feed_dict={self.s: s, self.a: a, self.r: r, self.s_next: s_next, self.d: d}))\n",
        "\n",
        "        # Count learning step and print current loss,  Increase epsilon\n",
        "        self.learn_step_count += 1\n",
        "        if self.learn_step_count % 5 == 0:\n",
        "            self.epsilon = self.epsilon + self.epsilon_increase if self.epsilon < self.epsilon_max else self.epsilon_max\n",
        "\n",
        "        # Update Target DQN, another way is to replace target with main Q network after certain step.\n",
        "        if self.learn_step_count % self.target_change_step == 0:\n",
        "            for i in range(self.n_layer + 1):\n",
        "                self.sess.run(self.W_target[i].assign(self.W_main[i].value()))\n",
        "                self.sess.run(self.b_target[i].assign(self.b_main[i].value()))\n",
        "        # Another way to update Target Network\n",
        "        #         for i in range(self.n_layer + 1):\n",
        "        #             self.sess.run(self.W_target[i].assign(\n",
        "        #                 self.tau * self.W_main[i].value() + (1-self.tau) * self.W_target[i].value()))\n",
        "        #             self.sess.run(self.b_target[i].assign(\n",
        "        #                 self.tau * self.b_main[i].value() + (1-self.tau) * self.b_target[i].value()))\n",
        "        # self.W_target_value += self.tau * (self.sess.run(self.W_main) - self.W_target_value)\n",
        "        # self.b_target_value += self.tau * (self.sess.run(self.b_main) - self.b_target_value)\n",
        "\n",
        "    def save(self):\n",
        "        \"\"\"\n",
        "        To save the model\n",
        "        \"\"\"\n",
        "        if self.save_path is not None:\n",
        "            self.saver.save(self.sess, self.save_path)\n",
        "        else:\n",
        "            print(\"Save Path needed\")\n",
        "\n",
        "    def plot_cost(self):\n",
        "        \"\"\"\n",
        "        To print the loss change after each training episode\n",
        "        \"\"\"\n",
        "        plt.plot(np.arange(len(self.l_history)), self.l_history)\n",
        "        plt.ylabel('Cost')\n",
        "        plt.xlabel('Training Steps')\n",
        "        plt.show()\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llKpBHm6c_U-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"\n",
        "Utility functions to enable video recording of gym environment and displaying it\n",
        "To enable video, just do \"env = wrap_env(env)\"\"\n",
        "\"\"\"\n",
        "\n",
        "def show_video():\n",
        "  mp4list = glob.glob('video/*.mp4')\n",
        "  if len(mp4list) > 0:\n",
        "    mp4 = mp4list[0]\n",
        "    video = io.open(mp4, 'r+b').read()\n",
        "    encoded = base64.b64encode(video)\n",
        "    ipythondisplay.display(HTML(data='''<video alt=\"test\" autoplay \n",
        "                loop controls style=\"height: 400px;\">\n",
        "                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "             </video>'''.format(encoded.decode('ascii'))))\n",
        "  else: \n",
        "    print(\"Could not find video\")\n",
        "    \n",
        "\n",
        "def wrap_env(env):\n",
        "  env = Monitor(env, './video', force=True)\n",
        "  return env"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RQ0zg-PTOOsT",
        "colab_type": "text"
      },
      "source": [
        "## 3. Training Step"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sJTxCtWxb1yJ",
        "colab_type": "code",
        "outputId": "6b5d4d08-489c-44c2-e03e-226ea7b6df1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        }
      },
      "source": [
        "env = gym.make(\"LunarLander-v2\")\n",
        "\n",
        "# Policy gradient has high variance, seed for reproducability\n",
        "env.seed(1)\n",
        "\n",
        "print(\"env.action_space\", env.action_space)\n",
        "print(\"env.observation_space\", env.observation_space)\n",
        "print(\"env.observation_space.high\", env.observation_space.high)\n",
        "print(\"env.observation_space.low\", env.observation_space.low)\n",
        "RENDER_ENV = True\n",
        "EPISODES = 5000\n",
        "RENDER_REWARD_MIN = 5000\n",
        "\n",
        "\n",
        "# Load checkpoint\n",
        "load_version = 0\n",
        "save_version = load_version + 1\n",
        "# load_path = \"LunarLander-v2.ckpt\"\n",
        "save_path = \"LunarLander-v3.ckpt\"\n",
        "reward_list = []\n",
        "\n",
        "Agent = DoubleQNetworkAgent(\n",
        "    n_obs=env.observation_space.shape[0],\n",
        "    n_action=env.action_space.n,\n",
        "    units_layer=(64,64),\n",
        "    learning_rate=0.0025,\n",
        "    name='ll11',\n",
        "    gamma=0.99,\n",
        "    load_path=None,\n",
        "    save_path=save_path\n",
        ")\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "env.action_space Discrete(4)\n",
            "env.observation_space Box(8,)\n",
            "env.observation_space.high [inf inf inf inf inf inf inf inf]\n",
            "env.observation_space.low [-inf -inf -inf -inf -inf -inf -inf -inf]\n",
            "\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kT6SE3BFpQo8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Agent.save_path = 'LunarLander-v2.ckpt'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PxxQnPOjwvST",
        "colab_type": "code",
        "outputId": "0dbf227b-a16c-4076-b330-e54ba362eea9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "for episode in range(1):\n",
        "\n",
        "    observation = env.reset()\n",
        "    episode_reward = 0\n",
        "    while True:\n",
        "        action = Agent.act(observation)\n",
        "        state, reward, done, info = env.step(action)\n",
        "\n",
        "        Agent.record(observation, action, reward, state, done)\n",
        "        Agent.learn()\n",
        "\n",
        "        observation = state\n",
        "\n",
        "        episode_reward += reward\n",
        "\n",
        "\n",
        "        if done:\n",
        "            reward_list.append(episode_reward)\n",
        "            if episode % 5 == 0:\n",
        "                print('Reward for episode %d is %f' %(episode, np.mean(reward_list[-5::])))  \n",
        "            break\n",
        "\n",
        "            if max_reward_so_far > RENDER_REWARD_MIN: RENDER_ENV = True\n",
        "                "
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reward for episode 0 is -129.633685\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrGCFdRUjvEa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "env = gym.make(\"LunarLander-v2\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qtHVAMMlkCu8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Agent.save()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}