diff --git a/README.md b/README.md index 4bfa1555..fde81b9c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ which demonstrate how to use Dopamine. This is not an official Google product. ## What's new +* **11/06/2019:** Visualization utilities added to generate videos and still + images of a trained agent interacting with its environment. See an example + colaboratory + [here](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/agent_visualizer.ipynb). * **30/01/2019:** Dopamine 2.0 now supports general discrete-domain gym environments. * **01/11/2018:** Download links for each individual checkpoint, to avoid diff --git a/dopamine/agents/dqn/dqn_agent.py b/dopamine/agents/dqn/dqn_agent.py index 2e648822..2c2c9808 100644 --- a/dopamine/agents/dqn/dqn_agent.py +++ b/dopamine/agents/dqn/dqn_agent.py @@ -102,7 +102,8 @@ def __init__(self, epsilon=0.00001, centered=True), summary_writer=None, - summary_writing_frequency=500): + summary_writing_frequency=500, + allow_partial_reload=False): """Initializes the agent and constructs the components of its graph. Args: @@ -142,6 +143,8 @@ def __init__(self, Summary writing disabled if set to None. summary_writing_frequency: int, frequency with which summaries will be written. Lower values will result in slower training. + allow_partial_reload: bool, whether we allow reloading a partial agent + (for instance, only the network parameters). """ assert isinstance(observation_shape, tuple) tf.logging.info('Creating %s agent with the following parameters:', @@ -157,6 +160,8 @@ def __init__(self, tf.logging.info('\t tf_device: %s', tf_device) tf.logging.info('\t use_staging: %s', use_staging) tf.logging.info('\t optimizer: %s', optimizer) + tf.logging.info('\t max_tf_checkpoints_to_keep: %d', + max_tf_checkpoints_to_keep) self.num_actions = num_actions self.observation_shape = tuple(observation_shape) @@ -178,6 +183,7 @@ def __init__(self, self.optimizer = optimizer self.summary_writer = summary_writer self.summary_writing_frequency = summary_writing_frequency + self.allow_partial_reload = allow_partial_reload with tf.device(tf_device): # Create a placeholder for the state input to the DQN network. @@ -516,13 +522,21 @@ def unbundle(self, checkpoint_dir, iteration_number, bundle_dictionary): """ try: # self._replay.load() will throw a NotFoundError if it does not find all - # the necessary files, in which case we abort the process & return False. + # the necessary files. self._replay.load(checkpoint_dir, iteration_number) except tf.errors.NotFoundError: + if not self.allow_partial_reload: + # If we don't allow partial reloads, we will return False. + return False + tf.logging.warning('Unable to reload replay buffer!') + if bundle_dictionary is not None: + for key in self.__dict__: + if key in bundle_dictionary: + self.__dict__[key] = bundle_dictionary[key] + elif not self.allow_partial_reload: return False - for key in self.__dict__: - if key in bundle_dictionary: - self.__dict__[key] = bundle_dictionary[key] + else: + tf.logging.warning("Unable to reload the agent's parameters!") # Restore the agent's TensorFlow graph. self._saver.restore(self._sess, os.path.join(checkpoint_dir, diff --git a/dopamine/colab/README.md b/dopamine/colab/README.md index 7f9bc0d1..30be1c77 100644 --- a/dopamine/colab/README.md +++ b/dopamine/colab/README.md @@ -20,6 +20,12 @@ In this [colab](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/load_statistics.ipynb) we illustrate how to load and visualize the logs data produced by Dopamine. +## Visualizing trained agents +In this +[colab](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/agent_visualizer.ipynb) +we illustrate how to visualize a trained agent using the visualization utilities +provided with Dopamine. + ## Visualizing with Tensorboard In this [colab](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/tensorboard.ipynb) diff --git a/dopamine/colab/agent_visualizer.ipynb b/dopamine/colab/agent_visualizer.ipynb new file mode 100644 index 00000000..4e52424d --- /dev/null +++ b/dopamine/colab/agent_visualizer.ipynb @@ -0,0 +1,199 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "agent_visualizer.ipynb", + "version": "0.3.2", + "provenance": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "TUQqwe_N6J-w", + "colab_type": "text" + }, + "source": [ + "Copyright 2018 The Dopamine Authors.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\n", + "\n", + "https://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y08iF10S6N-6", + "colab_type": "text" + }, + "source": [ + "# Dopamine Agent visualizer" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2cn16cOg6VLZ", + "colab_type": "text" + }, + "source": [ + "This colaboratory demonstrates how to use the agent visualizer functionality in Dopamine. It uses a pre-trained Rainbow agent on SpaceInvaders\n", + "and generates the video over 1000 steps of agent play.\n", + "\n", + "Note that it will save all the files to a temp directory in your runtime.\n", + "\n", + "To run, first make sure that your Runtime type is set to Python2 (under the Runtime menu above, select \"Change runtime type\"), as `scipy.misc.imsave` (which we're using for visualization) is deprecated in Python3.\n", + "\n", + "Then run all the cells in order." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "0uZ5ifEc6DKa", + "colab_type": "code", + "colab": {}, + "cellView": "form" + }, + "source": [ + "# @title Install necessary packages.\n", + "!pip install --upgrade --no-cache-dir dopamine-rl\n", + "!pip install cmake\n", + "!pip install atari_py\n", + "!pip install gin-config\n", + "!pip install matplotlib\n", + "!pip install numpy\n", + "!pip install pillow\n", + "!pip install pygame\n", + "!pip install scipy\n", + "!pip install tensorflow" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Q7BnRy2w6xEK", + "colab_type": "code", + "colab": {}, + "cellView": "form" + }, + "source": [ + "# @title Download an example checkpoint (Rainbow on SpaceInvaders)\n", + "!gsutil -q -m cp -R gs://download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.data-00000-of-00001 /tmp\n", + "!gsutil -q -m cp -R gs://download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.index /tmp\n", + "!gsutil -q -m cp -R gs://download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.meta /tmp" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "kpomDnVn9Ig1", + "colab_type": "code", + "colab": {}, + "cellView": "form" + }, + "source": [ + "# @title Generate the video\n", + "from dopamine.utils import example_viz_lib\n", + "num_steps = 1000 # @param {type:\"number\"}\n", + "example_viz_lib.run(agent='rainbow', game='SpaceInvaders', num_steps=num_steps,\n", + " root_dir='/tmp/agent_viz', restore_ckpt='/tmp/tf_ckpt-199')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "sIO7Y1V_xJ50", + "colab_type": "code", + "outputId": "26f88b27-12d4-41bb-a325-a561e511f2d1", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 441 + }, + "cellView": "form" + }, + "source": [ + "# @title Display the video\n", + "import base64\n", + "from IPython.display import HTML\n", + "base_dir = '/tmp/agent_viz/agent_viz/SpaceInvaders/rainbow'\n", + "video = open('{}/images/video.mp4'.format(base_dir), 'rb').read()\n", + "encoded = base64.b64encode(video)\n", + "HTML(data=''''''.format(encoded.decode('ascii')))" + ], + "execution_count": 10, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 10 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ha3tGluP_rAL", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 437 + }, + "cellView": "form", + "outputId": "0a1d2182-af13-4a1b-8d2a-8ab0d1c5c4e8" + }, + "source": [ + "# @title Inspect individual video frames\n", + "from IPython.display import Image\n", + "frame_number = 823 # @param {type:\"slider\", min:0, max:999, step:1}\n", + "image_file = '/tmp/agent_viz/agent_viz/SpaceInvaders/rainbow/images/frame_{:06d}.png'.format(frame_number)\n", + "Image(image_file)" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAGkCAIAAACqypkWAABVAElEQVR4nO3dd1wT5x8H8G8GhLD3\nBgFZIrLEAVRcrbgngmgdVZzYatVaW7VuW22d1da6autArQruxRAUFWXIFFnKkj1DEhJyud8fp/wo\nalVICJDv+9WXr8vD5bnvWf1447l7aIC6NIIgZF0Ckl8MBoMu6xoQQl0ZRgxCSIowYhBCUoQRgxCS\nIowYhJAUYcQghKQIIwYhJEUYMQghKcKIQQhJEbP5B7fNbu/9QsKaBKkV05K06/l7a//3rjPj+4et\n7v9jdbR6EGo7Wlu+3DwCPuSvekeLsI/1sRHQESIDHyBAMsRgMP4VMe+KjI+NEkmRdj3NI+Bd0SCr\noxhJ1YMRg2SoZcR8rLYcxXSECPtYHxsBHSHCMGKQDOFRjOwjQNr1YMQgGZLkUcy7fEg04FGM9GDE\nvEksFhcWFlLLxsbGTCbzv9f/EBcvXnRycrK0tGx7V10JHsXIPgKkXQ9GzJtqa2t1dXW7d+9eVlb2\n3XffffPNN23vc9y4cfPnzx85cmTbu+pKpHUtRlLtHQ0exXQNtbW13bt3r6ioOHLkyI4dO9LT0zMy\nMs6cOQMAQUFB2dnZHA7Hy8trx44dy5Yti42NVVRUVFFRuXTpEgDMnDnTwsLi119/pdPpYrF43rx5\nu3btEggEYWFh3333nZWVFdXPkCFDBgwYIOP97AD+6yjmXXBcjPRIox6MmDc1RcyhQ4f27NmTkJDg\n4OAwZMgQPp8vFAqnTp164sSJZcuWffLJJ+Hh4YcPH/bw8CAI4uXLlzk5OWVlZVFRUba2tqampv7+\n/iUlJSdPnuzTp8/p06cvX7586NAhS0tLGo3Ws2fP2bNny3pHZY/BYPzrLPRj/7q25VrMh6wv7Tj7\n2L+u0h4Xg8Pq2k1NTY2lpWVVVdW6detevnzJ4/E+//xzsVgcGBj4/fffP3v27MGDB+rq6g8fPkxL\nS1u1apWJicm+ffv09fXv3r1L9bBx48ZPPvmkX79+v/zyy/Dhw6l2Pp9vb28/YsQIMzMzme5fB9Km\nEyXU8eFRzJtqa2utrKxOnTq1ZcuWgQMHzpkzx9HRsX//V/8eXLx40cjIaPTo0d26dYuPj4+KihII\nBP7+/i9evHB1dQ0NDS0pKbG1tb1165aFhYWJiUliYqK+vj51LYYkySVLlpSXl585c2b48OGy3c2O\ngMFgyLoEJGUEekNVVZWWlhZBEA8ePDA0NExJSTEyMuJyuQRBlJeXEwQxbNgwBoORnZ3NZDIHDhxI\nEET//v2zsrIyMjL09PQIgujevXtOTg5BEAMHDjx48GBNTY26uvrly5cJguByuYcPHx4+fLjs9q8D\ngRbPKCEkV/r06WNubv7ixYuePXu6u7u7ubnl5ubeu3evV69ejx8/Njc3t7GxcXR0BABNTc2pU6fW\n19e36MHX13fVqlWHDx+uq6sDAG1t7VGjRkVFRc2dO1cG+9Mh4YlSF0fgidIbGhsbr127Nm7cOACI\ni4sDAHt7+xs3bgBA9+7dXV1dnz17VlRUNGTIkKioKH19/R49ehQXF8fExOjq6nI4nDFjxty4ccPb\n21tZWRkALl26JBQKdXV17e3tMzMzy8rKAMDX11emu9hRtPWmNer4MGKQDLW8o4TQf3uSlFRSVibr\nKlA7KSst9fP1VVJSaksnGDHoI4hEIq8B3rKuArWTJ4kJbT8KxrfeIYSkCI9iEFRXV5eWliopKVlY\nWPD5/Ly8PBUVFWrwWFlZWVVVlYGBgZaWlqzLRJ0SRoy8e/78+ZAhQ9TU1Lp37x4SEhIQEJCWllZW\nVpaSksJms52cnPT19TU0NMLCwlgslqyLRZ0PRoy8++2330aPHr1nzx6BQJCfn5+WlpaUlDRnzpwL\nFy4wmcxPPvnkxIkTVlZWz58/t7e3l3WxqPPBazHyrqCg4PTp05aWltTTOkZGRsrKyiNGjIiMjCwo\nKPD09FRSUho1atTDh/j8lHyhP89lXrwogX7a3gXq7MaMGRMWFnbgwIGXL19SLSYmJgKBoGkFFosl\nEAj4fP7jx49lVCNqb4r79yv+/HPb+8GIQTBq1CgbG5tPPvkkKSmJann27JmOjk7TClVVVTo6Omw2\nu0+fPjKqEbW78nKxnV3bu8GIkXc9evS4e/duTU3N3bt3x44dS70S5dy5c4MGDerRo0dISEh9fX1E\nRISHh4esK0XtilZdLTY3b3s/GDHyLjAw8MqVK0ZGRlOmTOndu/fUqVO7deuWm5s7ceLEgICA2tpa\nPT29oUOHmpiYyLpS1K7o1dUgiZEK+IxSFyfZZ5Ti4uPtejpKsEPUMTFiHyqPG/dw6tReP/+soqLS\n+n4YDDyKQQi1pHD6NDQ0iK2t294VRgxCqCVabW3jlCmEJC7AYcQghN7A4ZDNbim2BUYMQqglWl0d\nqaYuka4wYhBCLdE4HFBXk0hX+IwSQggAgFZfr7hlM43LAwDaixeSOorBiEEIAQDQnz1T3LsXWCyg\n0UgGQ9y9u0S6xYhBCAEAAIdDamryHj0mVVSARiPV1eFJYtt7xYhBCAEA0LhcYLFIXV1Soi8Gwsu9\nCCEAABq3HhQVSQUFyXaLEYMQAgAALpdUVga6hDMBIwYhBABAq+eCsrLEu8VrMQjJI1p5OfP6dSDJ\nphbG/RhSVVXiG8KIQUgeKRw/zlq7pkWjaNIkiW8IIwYheUSrrib69xds2ty8kZTQWJjmMGIQkke0\nujrSzIzw9JT2hvByr7wLDg62t7e3t7ffvn07APz888/29vYDBw7kcrlCoXDMmDH29vbffvutrMtE\nklZXR6pL5hGB/4ZHMfKupqbm66+/HjFihJqaGofD2bNnT2ho6Pz58y9cuKCqqlpUVHThwgUPD495\n8+Z1l8JRNJIVGocjNjZqhw1hxCAIDQ1NSUlZvHhxTU2NlZWVu7v7tGnTIiMj9fT0/Pz8HBwchg4d\nmp6ejhHTQTDu3qU/f97GTug52eJ2mU8CI0bemZubjx07NjU11dfX948//qAanZ2db926paenp6io\nCADGxsYvX76k5lHCd/fKHHvWTFpJSdv7Eevrtb2T98KIkXejRo0CALFYrKWl1TQ9W0VFxZtr4jxK\nHYJYTONwBD/9JHZza1M/dDrh2rYePgxGjLwTi8V0Or2mpoYgCF1d3aqqKrFYHBkZOWjQIAUFhbS0\nNKFQeOXKlcWLF8u6UgQAQOPxgCCIvn2Jfv1lXcsHwYiRd/Pnzy8vL8/NzR0+fLiTk5O6urq3t/fj\nx4/T0tIUFRUdHByKi4u1tLTs7e1lXSkCAIAGPojFoNz6iUfaGUaMvFu8eHFpaSkADBkyBACuXr36\n+PFjbW1ta2trAIiPj8/Ly8N86Tho/AYgCGmM9JcSjBh55+zs3PyjlpbWsGHDmj7a2dnZSWJiYyQx\nfD6IxWQbpk9rZxgxCMkY/flz4PE+dOXMZ0CSeKKEEPogtOpqlT7uwOd/xHeUlUGh0/zN7TSFItQl\n0aqqoKFBsHo1qH7opCJiUxNSUZLvvpQqjBiEZInG4wFJNi7+sn2eGGp/+BgkQjLF4wEAKYXXzXUQ\nGDEIyRKNywVlZaDTZF2ItGDEICRT1Eu5ASMGISQFtPp6amo0WRciLXi5F6G3oPF4Ks5OtMpKqW+J\nIMS2tl34KAYjBqG3oJWX0YqLRaNGgRJb2tsivLy6bsJgxCD0NjQeH0iy4cAfpJaWrGvp3PBaDEJv\nw+UCgDSmLpM3GDEIvQWNywUFBVJRUdaFdHoYMQi9DZfbiZ5m7sgwYhB6C1p9fSd6mrkjw8u9CHg8\n3ujRoz/77LPvvvvup59+OnTokIGBwc2bN1ks1tixY7OyssaPH79jxw5Zl9lK9NISpTlz4PVriT8Q\nrayMVO3Kw1XaDUYMgtOnTz958qR37951dXX79u27cuXK3LlzQ0JCVFRUysvLb9y44e7uvnDhQuo9\neJ0OPf0p484dsa3dx90YZjKJQYOkVJJcwYiRd1wu98CBAwEBAQCQnp5uZWXl4uJCzaOkr6/v5+dn\nY2Pz2Wefpaend9KIAW49qa3Ne/DgYw9JSCb+7ZAAvBYj706dOmVpaWljY9O8sVevXsXFxQCgoKAA\nAIaGhsXFxdQ8SrKpsg1o9VxQUSEVFT/2P6Dj3w4JwN9Eebdv3z5XV9ekpKTMzMyS1xOAlZeX0/79\nbz6NRuus8yh19YeAOjg8FJR3AwYMSE9Pz8jI4PF4YrG4srKSIIjw8PDBgwczmcyUlBSBQHD58uUv\nv/xS1pW2Eo3HJfHekOxgxMi7ffv2AcDu3buLioomTpy4a9cuLy+vJ0+epKWlsVis9evXFxUV6enp\ndd55CKgTJVlXIb8wYhAAwKRJk6jZZq9cuRIfH6+trd29e3cASEhIyM/Pt7W1bc9imNHRjGtXJdUb\n4/590sBAUr2hj4URgwAAzMzMqAUNDQ1qzjaKtbV1+99IUvjjAPPqVQm+AbvR01NSXaGPhRGDOhxa\nVZVwwcLGoCBJdUjq6UmqK/SxMGJQh0Orria7dRO/PrBCnRretEYdDEHQ6urwLS1dBh7FyIvdu3cX\nFBQ0fXR3d6dG9LYD1qZNjJh7H7q2WEwrLsaI6TIwYuSFqalpRUXF8ePHV61ade3aNTqd3m4Ro3D2\nDPB4pNoHz3Zoayu2s5dqSajdYMTIC19fXyaTqauru3DhwhEjRqxfv759tksTiYDPb9j+MzFmzAd/\nh0YqKEizKNR+MGLkS3R0tKWlZUZGRvttskEAAgFoaeEb5OQTRowcGTly5JEjR/z8/MzMzEJDQ9tp\nq0IBTSDoqhM2o/fCiJEjiYmJCxYsuHz5cjttjyQBgCYQQEMDqH/ohRjUxWDEyJGKior09PRRo0a1\nw7YUf/mFEREOVMSIxaQqRoycwoiRIyoqKmvXrr1+/ToADBs2bNWqVdLaEkkqnDlDKymmnj8k+vQh\nNTWltS3UsWHEyBFdXd2vvvqKWra0tJTehmgiEY1TJ9i1u9HXV3pbQZ0CRowccXR0XL58eXV1NQBo\naGhIcUsiEXA4OHwOAUaMXElOTh47dmxdXR2fz58zZw71ppiPIxbTqGkS/xONw6HV12PEIMCIkSu5\nubn+/v537tz54Ycfrly50ooe6FlZKhMmvH89MQliMamBN6oRRoyc0dXV5XK5BgYGfD6fagkKCqLi\nJigoaOXKlZs3b6bmUQoLC1NSUho5cmRWVta4ceP27t0LACSbTbi7f8iGSB0d0sRUejuCOguMGDmi\nr69fWVlpY2MzePDgwMBAqnHGjBnr16+Pi4sLDAycPXv2gQMHbt68OXv27JCQEDabXVtbGxUV5eLi\n8tVXX1lbW5Pm5vzTZ2S7F6hzwYiRI3Z2dp6entOmTRMKhWw2m2rs169ffHz8o0eP2Gx2RkaGlZVV\nz549p06dGhkZaWBg4OfnZ2Fh8emnn6alpXXWeZSQTOH7YuTIvXv3NDU1g4KCYmJiFJo9Z3jx4sWN\nGzd6eXkxX09O5ujoSE14wmAwAMDQ0LCkpKSTzqOEZAsjRo6MGzfu/PnzAODv7z9r1qym9o0bN1ZW\nVp47d47H41EtZWVl9GYTlZEkSafTO+s8SkimMGLkSH5+/p9//nnlyhV3d/cJr28MURMPCIVCkiT1\n9fUrKipEIlFYWNjgwYMNDQ2TkpIEAsGVK1cGDhwo09pRZ4XXYuRIWlra2bNnJ0+evGTJEicnJ6ox\nKCiooKAgLy/P29vbwcFBX1/fw8MjNTWVmkdpw4YNBQUFBgYG7TzPCeoyMGLkyIgRIx4+fPjnn39+\n+umnX3zxxZ49ewBg+fLlFRUVANCzZ086nX7p0qWkpCRNTU0rKysASEhIKCoqouZUQqgVMGLkSFZW\n1o4dOyIiIrS1tV1cXKjGHj16NF9HXV19wIABTR+trKyorEGodTBi5Eh+fr6ent4///zTp08fFkti\nE6Eh9B8wYuTIoEGD8vPzY2Ji1NTUamtrvb29ZV0R6vrwjpIcefLkyZ9//nn69GkVFZWjR4/Kuhwk\nFzBi5EhBQYGfn5+Ojo6SkpKsa0HyAiNGvsTGxjb9ilA7wGsxcsTHx2fjxo0pKSmPHz+OjIyUdTlI\nLmDEyBE2mx0XF3fz5k0jI6Pi4mJZl4PkAp4oyYucnJyJEydOmzatqKho4MCBd+/elXVFSC7gUYy8\n+P333ysqKhQVFVesWLFr167Zs2fLuiIkFzBi5MjevXv5fP7evXsxX1C7wYiRIydPntTQ0MjMzNy1\na5eDg4OPj4+sK0Jdn9SvxWzdaibtTaAPoaend/ny5RMnTnC53D/++APvKKH2QZNsd1u3mn3/fcG7\nPqL2RxCEBHuLi4+36+kowQ5RR/YkMcHNyUlFRaXVPTAYDEkexVCB0nTYgvmCEJL8tZimlMF8QQhJ\n8iiGypTmRzF4IabjW7NmjbGxsbGx8aFDhwBgw4YNxsbGrq6utbW1QqFw4MCBxsbGixYtknWZqLOS\n8OXe5scvLRIHdUxOTk6JiYn79u07fPhwTU3NoUOH7ty5w2azQ0JCLly4wOfzHz16dOrUqaysLFlX\nijolqdy0pmIFT5c6BT8/PwCwtLQEAGoeJVtb2ylTpkRGRhoaGvr5+Zmamg4bNiwtLc3GxkbWxaLO\nBx8gQECS5MaNG3v27NnU0rNnz9LSUgCgpjrR19cvLS3FeZRQK0jlcm/TMp4ldQo7d+68f/9+QkJC\nQcGr/3elpaXUJG0UsVjMYDBwHiXUChK+aQ3/vtwLmDId3r1793744Yfjx4+bmJjo6+uXl5cLhcLb\nt28PHjzYyMgoISEB51FCbSHhoXeoo3nv0LspU6Zcu3bN3d0dAK5fvz5q1KjKysrMzMzU1FQWi9Wz\nZ09HR0eRSPTgwQPAoXdyRiJD7/AZJXm3adOmpUuXUssKCgohISFpaWnq6urUBeAnT54UFxdbWFjI\nsELUqWHEyDsbG5vmt4rU1NT69+/f9LFbt27dunWTRV2oi8A7SgghKcKIQQhJEUYMQkiKMGIQQlKE\nEYMQkiKMGISQFGHEIISkCCMGISRFGDEIISnCiEEISRFGDEJIijBiEEJShBGDEJIijBiEkBRhxCCE\npAgjRt79+eefenp6Q4YMoT7+8MMPenp6jo6ONTU1QqHQ09NTT09v3rx5si0SdV4YMfLO09Nz8+bN\ntbW1AFBdXX3kyJFHjx5pamqGhIScO3dOLBanpqb+888/mZmZsq4UdUoYMfLOzs7Ow8ODWn727Fn3\n7t0tLS39/f0jIyOTkpImT55sYGBAzaMk2zpRJ4URg97CwcGhrKwMAGg0GgDo6emVlZXhPEqoFTBi\n0FuUlJQwmf9/r7NIJGIymTiPEmoFjBj0f4aGhmVlZQKB4NatW4MHDzY2No6LixMIBNeuXcN5lFDr\nYMTIu9DQ0FmzZmVlZQ0YMMDIyMjCwqJPnz7nz5+fNGmSv7//zZs3Bw0aZGZmZm1tLetKUaeEk5zI\nOy8vr4MHD1LLCgoK586dy8jIUFdXp+ZOevLkSWlpqbm5uSxLRJ0ZRoy809PT09PTa/qoqqpKzQxJ\nMTMzMzPDKYNR6+GJEkJIijBiEEJShBGDEJIijBiEkBRhxCCEpAgjBiEkRRgxCCEpwohBCEkRRgxC\nSIowYhBCUoQRgxCSIowYhJAUyTJidu7cKcOtI4TaAR7FIISkCCMGISRFGDHonQiCGDZsmKamppub\nW319vazLQZ0SRgx6p8TExMzMzNTUVBqNdvv2bVmXgzoljBj0TmVlZc7Ozqampl5eXqmpqbIuB3VK\nGDHo/QYMGPD06VOcRwm1AkYMer/k5GRjY2OcRwm1giwjZtmyZTLcOnovFotVWFgoEAgSExMdHR1l\nXQ7qlHAGAvROAwYMqKmpcXV1LS8v//PPP2VdDuqUMGLQOykqKt69e7e8vFxTU7P5RCgIfTiMGPRf\njI2NjY2NZV0F6sTwci9CSIrwKAZ9BC6Xe+/uXVlXgdpJdlamm5NTGzuhSaQU1GERBCHB3gQCgQSf\nJLh//76np6ekesNNSGMT2traNFrrU4LBYMgmYtw2uyWsSZDJpuWNZCNGskpLSw0MDHATXXgTDAZD\nBtdi3Da7Nf2K5MqJEydWrVoFAM+ePfv888+XLl1aVlYGAMHBwQEBAdSPWq2qqiogICAgIIAab5Wd\nnT1z5sylS5cWFRUBwPnz55t+1BZHjx4NCAj48ssvASAiIiIgIGDdunUCgQAAtm3bFhAQsH///rb0\n/+TJk4DXwsLC8vPzAwMDly5d+uzZMwCIi4sLCAgICgpqy78c5eXlX375ZUBAwPbt2wEgNzc3ICBg\n6dKlKSkpTTu1YMECaqfaDi/3onZSU1Pz1Vdf3bx5EwC2bNmira1dU1OzYcMGoVC4dOnSYcOGXbhw\nITg4uNX983i8srIyf3//kSNHAsDPP//MZrN5PN7atWvFYvHixYuHDh1669attgzwuXPnzqZNm0aO\nHOnh4UGS5DfffNO7d+/o6Ohjx45lZWXt3r178uTJy5YtS0tLa/UmjIyM/P39/f39Hzx4wGKxfvvt\nNy6Xq6+vP2fOHJIkp0+f3r9//7y8vN27d7d6E2fPns3MzJw8efKuXbtyc3NXr14tEomUlJS++eab\nFjvV6k00196Xe5sfvODpklz55ZdfRo0aRT1OGRISUlFRER8fP3fu3KioKEtLyy+++KKioqKND1sK\nBIKqqiobGxtqE5mZmc+fP/fz87t3756+vn5gYGBDQ0NbNhESEuLg4NDY2Ojs7PzixQsOh7NixQo1\nNbXIyEgulzt69OiJEyf6+/unpqb27NmzdZswMDAYP37806dPuVyuq6trYGDgyZMnu3Xr5uHhkZCQ\nIBAIgoKCdHV1g4ODly9f3rpN1NTUWFpaDhw4UFlZmcPhPH369OTJk1paWv369cvNzW2+U/Pnz2/d\nJppr76OY5pmC+SI/ysvLT506tWjRouaNnp6emZmZAKCtrQ0Atra21MfWYTAYWlpad+/e9fDwKC0t\npRpdXV1fvnwpEomaNkGdcbTa/fv379+//9lnn3E4HKrFy8uL6lNSmwCAyMhIHx8fVVVVqls9PT2B\nQFBbW6umpsZkMm1tbauqqlrdeWBg4KFDh/T19XV1dZ2dnalGHR0dAMjLy2uxU22HJ0qoPezfv3/w\n4MH19fWNjY3V1dVUo6TO9ilGRkaXL1/+888/Bw0aFBUVRTUKhUKSJCW4lZUrVx4+fLhnz55hYWFU\ni2T3ghIZGTlo0CCJd0sJDg4eN27cnTt3ioqKmjJdJBIRBMFkvjqtkeBOteuJUotLvNRHPJaRB1wu\nNyYmJiIioqSk5ODBg2w2m8/nx8bGurq6KisrFxQUAEBMTIybmwRuAigoKAAAtYnk5GQbGxt1dXWJ\nbILNZisqKgIA9StJkiKRKDIy0s3NTVlZOT8/HwDu3bu3ePHittTP5XIjIyN//fVXAKC6JQiCzWYb\nGRlVVVU1NDQkJCSYm5u3uv/a2lo3N7cBAwaYmJhQQxD4fH5JSYmCgoKVlVXznWrLXsiM22a35v/J\nupyuj+hIkpKSnJycCIJYuHChj4+Po6PjunXrhEKhmZnZl19+aWxsHBcX1+rOL126tGLFiuXLl6uo\nqOTm5n799ddDhw51cXFZuXIlQRCWlpaLFi0yMzOLiYlp9SYeP36sr6+/cuVKPT29yspKLy+v6dOn\n6+npnTt3rrCwUF1dfc2aNaqqqrW1ta3eBEEQoaGhLi4u1PKmTZs8PT3HjBkTGBhIEMSAAQNmz57t\n4uLy119/tbr/bdu2OTo6rlmzRkVFJTU1dfHixX369BkwYMC0adMIgmi+U23ZCwq089C7t2YKHsVI\nFdGRxsVUVFRcunRp9uzZ1dXVhw8fJghiyZIlbDb78ePH4eHhpqamn3/+eas7T0tLu3z5MgB4eXlR\nz4gfPXpUKBQGBQWpqak9efLkxo0bhoaGs2bNavUmCIK4cOFCTk5Ov379Bg8enJ2dfe7cOQ0Njfnz\n59Pp9GvXriUnJ7u6uvr4+LR6EwAQERHBZDK9vb0BgMPhHDt2jMvlzp49W19fPy8vLzg4WFtbe+7c\nua0eEcfn80+ePFlRUeHs7DxixIiqqqqDBw8CwIwZM4yNjVvsVFt2BNp/6B1GTPvrUBGD5M37I2Z8\ngH77lIKk5PyJYlmXgOQXg8F45+VeCYZLfk9TADBPK5RUh53XxF5WAHAhJVfWhaD/o95JTJ2YIIlj\n4nEKkkMeHh7UeHkAYLPZfn5+HxUxR44cGTdunK6urnSq61LwZQ5IHsXExADAsGHDbt26xeVy7969\nW1dX9/jx44qKCltbWw6HU1xc7OvrS6fTIyIiKioqhg4dqqamFhoaCgA9e/Y8deoUn88fM2ZMY2Nj\nfHy8pqamj4/P3bt3hUJhRUWFn59fW55O7mIwYpA8arpXQqfTKysrly5deunSJR8fHz8/v/DwcDMz\nMwAQiUSVlZXHjx/X1dW9evVqjx49Ll68aGlpWVRUVFhYeOfOHWtr6yVLlri5uT169Oj48ePbtm17\n+fIlg8FIS0vbuHGjTPevA8GIQeiVIUOGnDp1ytvb++TJk2fPns3IyIiJiRkyZIiFhcW3335rbGxs\naWm5bNkyNze3p0+frl69urGxUUlJaciQIQAQGRkJAMeOHWOz2QsXLpT1rnQg7fEAgXlaIV7rpVxI\nycVrvR2Wra0tAGhra2tpaTW9nyknJ+fp06fz589fsmSJhoYGFUNNXxEIBCkpKXp6etRzjxYWFt26\ndZNJ8R0WHsUg9E5mZmbq6urLli178eJFbm7uqlWrVFRU7ty5Q6fTExMTbWxsOBzOokWLGhoa1NTU\nAODw4cNKSkosFkvWhXcgGDFIflFP+lDP5rBYLOq1b6ampnQ6nc1m6+np+fn5BQUFjRw5ctCgQd26\ndfvrr78AYN++fampqatXr962bdvs2bMnTpwIAGfPngWAkJCQysrKM2fOyHS3OhYa3rTu2nDoXbsZ\nPXr0uXPnlJSUZF1IByKbF2si1CXZ2tq2/aGergdPlBCSjJ07d8q6hI4IQxchJEUYMQghKcKIQQhJ\nkdw9SYHvT0EyxGAwZF1Ce8OjGISQFGHEIISkCCMGISRFGDEIISnCiEEISRFGDEJIijBiEEJShBGD\nEJIijBiEkBRhxCCEpAgjBiEkRRgxCCEpwohBCEkRRgxCSIowYroIkiTzX2tsbHxzhcLCQrFY3Or+\ni4uLqZfs79mzp8WPqqqq6uvrm7ckJiZGR0eXlJT8x7v4CYIoKioCgOrq6r///rvVhaEODiOmixAK\nhVZWVsOGDXN1dd22bdubK7i5udXV1bW6/xcvXuzbtw8ADh061OJH33//fYsoSU1NffDgQV5e3q+/\n/vquDktKSry8vACgtrYWZwXpwvD14F2HkpJSRkbGyZMn165du2bNmpqamsOHD3O5XB8fHwaDwePx\ntm3bNmTIkLKyMg8PDysrq02bNq1du/bw4cMCgYDL5To6Oj5//ryiosLGxmbq1KlN3YaHh9+7d6+h\noYH66O/vDwChoaFJSUkAMGPGjPj4+JKSEl1dXbFY/PLly4qKCm9vb2pOIh6Pt2nTJpIkV6xY8fLl\ny0ePHk2dOjU9PT09PV0oFNbW1m7YsMHb23vUqFEAkJOTc+LECQBYunSphobG77//ThBEZWXl1KlT\nbWxs2v/3E0kEHsV0NWKxmJpqY/ny5VevXhWJRNOmTePz+TQajcViMRiMkydP5uTkkCS5adMmADh8\n+PDOnTtZLNa1a9fWrVtXV1e3YMGCzMxMqjc+nz9t2rTKysqwsDAAIEly8+bNAoFg+vTpfD6/srKy\nvr6ewWAwmUwmk3nhwgVquvhHjx7dunULAJKSkhISEkJCQlatWpWdnR0cHAwAT58+PXfuHJPJpEqq\nqanZv38/AEyfPv3p06fR0dHLly8HgMWLF0dFRcXFxb15aoY6EYyYroPP51taWn755Zdz5swBgHv3\n7s2ZM2fEiBFmZmZMJpPNZi9dupSa472FnTt3LlmyBAAWLly4Y8eOMWPGxMbGUj/KysrS19ffs2fP\n9u3bm9YnSbKxsXHAgAE//vhjr169XFxcRowYQR2JfPPNN+vWrWOz2dSaFhYWp06d2r9//7Vr11ps\n1MvLS11dfdWqVdRs0M+ePSsqKvr7779Pnjx55swZKihPnDixaNGi/Px8yf9mofaCJ0pdh5KS0qFD\nh3788cfq6mqq5ffff2ez2Uwm860XgJuYmpo2X2gxJbOioiKNRms+yaGSktLy5cuDgoJEItHt27ff\n2hXFyMiIzWb379+/sLDwv4sXCoU6OjqKioqGhoaNjY0kSXp4eODk0F0ARkzXQaPRPv30UwMDgyFD\nhvzwww8AsGfPHnd3dy6Xy2QytbW1m69cVVX1gd3W1dURBNFi/S1btqxZs2b69OmRkZH/8d2cnByx\nWHz58mU7OzttbW3q/Outm1ZTUyspKeHz+SUlJerq6jQaTVFR8QMrRB0ZRkxX06tXL0dHx9u3b8+d\nO3fYsGGjR4+OjY2Niorq1auXr6/vnDlzevTosXLlyg+cGtXR0ZEkyeHDhz979szCwoJqbGhocHR0\n9PT0jIqK8vPz09XVPXLkyFvviJeVlfXt27e4uHjBggUuLi4VFRVjx4598ODB0KFDlZSUGhoaZsyY\nQZ27WVhY9OnTx8PDo7a2ds6cOThza5eBk5z8X25u7uPHj9XV1UeMGNHUGBUVVVJS4ujoSF0yAIDE\nxMT8/Pxx48ZJvdaPIRaLL168OGHCBABITEwUCoX9+vW7fv06l8tVVVUdPnx4VlZWUlKSra2tiYlJ\nZGSkrq5uZWXlpEmTwsPD3dzctLS0EhMTNTQ0rKysHj9+rK+v361bN6rnzMzM5ORkXV1dkiQHDx58\n/vz5CRMmXLhwAQCMjY09PT3z8/MfPXrUrVs3kiSNjIzMzMyys7N5PJ6pqemTJ0+qq6tJkvT19QWA\npKSkrKwsXV1dJSWl/v37x8XFvXjxokePHoWFhT4+PjU1NdRF5QkTJjAYjKioqIEDB5aUlOTk5FC3\nt7sAOZzkBCPmFZIke/furampGRcXFxISMnToUKp9/PjxcXFx8+fPX7t2LQA0NjZSVxZKS0vbr2jU\nVchhxOCJ0iv5+fmlpaUPHjzYsWPHxYsXmyImNDSUuhFLiYmJodH+n8tPkpJKysrau1YkI2WlpX6+\nvs2vfKP3woj5P2VlZRaL5ejoePPmzbeuQJLkhg0b1q1bFxgYCAAEQZSXl3sN8G7fMpHMPElMwNlE\nPxZeVGvJ1NS0pqbmrT8KDw9XUlJqui5AjT1tv8oQ6oQwYlqKjY3t3bv3W390+/bttLS0IUOGVFdX\nf//994qKitbW1u1cHkKdC54ovaKpqcnhcP7555/g4ODx48cXFxdfvnx53rx5ubm51JXd3Nzc5cuX\nz5w5s7a2dsyYMUFBQS16EAoEJEGQsij+rWg0GkuJLX8X9FHHghHzioaGxjfffLNjxw51dfVZs2aV\nl5c/fPhw3rx5u3fvjo+PB4A9e/bs2bNHX1+/rq6uT58+JiYmzb8uJgglJkNJTVVG5b+FSCTiNggU\ncAAbkim5+zdOspfr4uLj7Xo6AkCjUKjKVupotyRrOBxFFt7+kJgniQluTk4qKiqt7qGj/QlpB3gt\npgN59OjRu3707NkzknzPSVhjY+ODBw84HI6k65IZem4OMyKCkZws60JQ6+GJkoTR9+6FD/grQW7f\nTv77oaHi4mIfH5/c3FwtLa2mxidPnqSnp0+dOjU6Otra2vq//w3cvGkTi8U6fOjQkaNHW11/h6Lw\n99+KP/8sGjGCf+68rGtBrYQRI2lhYbTr19+7FrluHfw7YiIiIj6fNi0yMnLixIm5ubk7d+xwdnYu\nr6i4dvUqkCQ13Cs6OvrkyZOzZ8/u3bv3wT/+yMrO/uyzz0aOHAkABEHk5+f/eezYqm+/ffnypbGx\nsZT2D6GPgidKHUXMvXur16y5ExkJAOvXrVv13XefDRs2bNiw2XPmTAkIiI+P5/F4Bw8e3LRp057d\nuxsaGk6dOvXdd98FBwdTJ1AEQTCZTADQ19cvLy+X8c4g9BoexXQIBEFUVFY2CoUvi4uFjY1qamrU\ni1cqKyvpdDr12HFZWZlDjx76+voWFhYcDse9Tx99fX1tLS2xWEydQFFZ03HumiMEeBTTQaQkJ9fU\n1Gzbtq2qqio7M7PpDVI0Gq3pKi+LxeJyuQDA4/EUFBRavO6AwWBQN8tKSkoMDQzat3ypUzh4UHn0\naFazN++hzgKPYjqEsPDwbdu2ubq6xj58GBER0a1btzVr1hgaGk6YOPHGtm3UGGJ9ff3Kysr169eL\nxWI1NbUWPTAYDFs7u++//55bX29gaCiLnZAielYWIzKixQVy1CnguJg2eXNcDC0zE2pr3/tF0tkZ\nmg2Ke/78ubm5OYPBEIlERUVFpqam6enp+np6BoaGWVlZSiwWSZJm5uZ8Pj8rK8vOzo7FYhUUFJib\nm+fl5XUzNwcaDQBEItHT9PTu1tbKyspUt519XExlwrqq5J9VzUZY3LBS/G2/aJIvX6YzLuG4mFbA\noxgJI21tW/EtS0tLaoHJZFLvgurVqxfV0nx+D2VlZWdnZ2rZ3NwcAJpeHEV9t5eTU6uq7qAEelBn\nD3St96+JOiyMGNQJ5E+Dqj6gYQ54K77Twcu9qBMQagHPHAS6sq4DfTyMGISQFGHEIISkCCOmoxCL\nxd98802Lxpra2oKCAgA4eeoU8bZZRJoLDQ396ssvBUKhtEpE6ONhxPzfF198oaSkpKenl5ub29Q4\ncuRIFou1ZcsWAIiLi1NSUmKz2Z9++um7Opl0dqLyFvZ7/yuqazk7YnJy8rl//mmaTFosFpMkmZ2V\nFRYWRpJkD3t7Oo1GtVMrkCTZYuoiJSWlyqoqkUgkid8MhCQD7yi9Ul5efv369ZSUlPXr1584cYKa\nTREA1qxZ0717d+ovs5KSUmpqqrm5uZ2dXUZGhr29/Zv9iMQikfj9f8nJNwb6h4WF/bBu3e1bt2xt\nbU+dOnXun38cHByUVVSuXrmiqKj4+PFjJyenzZs3pyQn9+3b9+tly0aPGmVgaGhkZNQ02/Tw4cPf\n9WLzzqUmpKryTLmyowrMlHUpqM3wKOYVHo+npqZmY2MzaNCglJSUpnZPT089PT1q2dHR0draWkFB\noWlsmwSlp6V9/vnn8fHxYrH41s2b5y9c2Lxli4+Pz9x586ZNmwYAFZWVtTU15y9cePHiBbe+Xltb\n+/jx4/UczlunYezUGrL4NZer6u/XyboQJAEYMS15eXk1na281bFjx2g0mrm5OZ/Pf/z4sUQ2yuFw\n0tLSduzYkZySwuPxVFVVm8/WRKnncPT19QFAS1ub39BgbGICAAoKCu99VVUndn0krPgFjs6RdR2o\n9fBEqaUnT55QTzm/VXJy8tKlS//44w/qQKZPnz4S2WjMvXvjxo/39vYWCgSPHz3icDhcHo8Ui5kK\nCg0NDdQ6Wtra+fn5DQ0NpSUlbx3DLhKJCIIQCoUqUjjIko2XxpAoBkMtgEhZl4JaCSPmFQUFhfr6\n+rKyskePHjk6OgoEgtLSUmqQfpO6uropU6bMmDFjypQp7+rHWM3EWsfmXT9twqQrNP+Ylp4+c9Ys\nE2NjY2PjK1euzJw1a+6cOc7OzkGLF/+yfbu6mpq+np6mhkZvd/cvvvhi1KhRysrK1Ombvr4+vD7e\n2bNnT3x8/IL5848cOaKq2oFeVI7kGT4G+X+jRo1KTU2trq6Ojo4mSXLBggWxsbELFiy4ePEiAIwb\nN27s2LFjxoyxtrZWUlIKDQ21tLTE14NLQ8n2oqIN+eoDNVT6qhX/XKg5Qktxx5Gy4v3aur6W1vgY\nZCeDRzH/FxwcXFpaqqio2K1bN4FA8M8//wDA2rVrly9fDgDKysqampoZGRnUyi0mOUEIvRVGzP+p\nq6urq6tTyywWizpLahElzZ97Rgi9F0YM6kz+ST8TW/Sgr0k/P4cAWdeCPgjetEadyd386MMJB6Pz\nomRdCPpQGDGoowi/TZ83R+HHLR90ZL0oJ8fwUezqvBdSLgq1FZ4oSdi9u7SXxe9fbcxokv3vwStc\nLtfO1jb3+XPFZi/czM3Nzc/PHzRo0IoVK3766SdqGpO3Ighi2tSpwsZG7wEDln79det3QHYyn9HO\nnqb3die/GPX+lYUk2SAWN3bhYYddBUaMhP3884fM1Aa5uaTpvyPm7t27jr16Pbh/f+CgQQ0NDTdv\n3rSwsEhNTY2OjlZVVfX392cwGOVlZdF37w4ePFhTU/Px48eVlZWOjo7UZWk6nX74yBEVFZWAgACx\nWNxifgKEZAX/IHYUERERW7ZsCQsLA4BV335bVVmZkZGhpKSkqqKiqqJy8uRJgVC4dOlSPp//9dKl\nQqHwy8WLq6urv1u1ivo6jUZTVVUlCILEfEEdCf5Z7ChKS0p69er14sULkUjU0NDwxezZ/v7+FhYW\nvZyc7Hv0AICS4mIbG5vPP/9cT1+/rq6uT58+06ZN09HVbf4Y5Ong4M+GDZPdTrRJd6gfD0WD4L+m\nslRlgg4LVJsdfFeLRCVCIUeiE0sgCcITpQ4hLy8vLj4+YMqU9PT00tLStx6GkCRJjQ1lMBhisZil\npAQAtNeTQAJAbm5u5J07B//4oz0rlyAXqNGBFyqgBvDOOQcCrDS8VA3N9DX2vs7VoNyca1VVS41N\n1v/7aQ/UQWDEdAgRERF79+wZ+umn165dox5fCA8PZzKZxsbGGRkZZWVlAGBgaPjs2bPo6OiXRUVN\nQwSbiMXi5cuWLQoKyszKsre3f/NB7a7hEAT+DWM+B32cWbezkMeIuX///m+//bZp06am2YskaOo0\nsm+/96+mrvGvj0ZGRh6engAwYMCA2NjYTZs3nzp50sHBwcrKyt7OLjkpaeSIEUos1toffrh29eqG\njRtZLNaI4cMBYOTIkdQhD0mS/fr1e/ToUVJSkp2dXVeNGNTpyGPEGBsb02g0JyenL774YvXq1QYS\nnQHa3781/7oOHz6cWlBTU6Pe2vnVkiVUy6wvvmhazc7Ozs7Ojlqmrrn4vP4ig8FY9d13ra0aIWmR\nx8u9FhYWf//99+3bt2/cuOHg4LB+/Xo+ny/rohDqmuQxYsRi8Z49e2bOnKmkpLR169Zz585Rb/lH\nCEmcPJ4o3blz5+jRo3v37h08eLCioqJYLKZe4HTx4sVLly6pqalt3769aYjtzp0709LSqJfFAMDv\nv/8eFxdnaWm5Zs0aWe6D7DSKaFamigDwwwbR3Pntdav44Fw4OBzGGwCeC3Y28ngUo6Ojs3HjRh8f\nn8bGxr179y5cuNDY2JjH4y1YsMDc3Pzq1avHjh1rWpnL5ebk5CQlJQFAXl7e119/7eTktGPHjpiY\nGJntgGyRwOEAhwPtOl+TUBF4yiBgteMmkWTIXcSQJFlYWPj06VOBQFBfX3/v3j2qvby8XFVVdd26\ndcuWLUtOTm5af+3atUOGDKGWU1JSfHx8lixZMm3atNTUVInXduPGjTerpUbWJSQkvPc14KmpqaEh\nIZWVlRIvDKFWk7sTpcjISOqUhzrT+fqNJwZ79ep14MCBt3737t27jo6OTR9JknzzOnF+/iYOJ+69\nZdjaHlFQ+Ncs8Pn5+VOnTn327FnTnCoAkJCQkJKSMmvWrGfPnjm7uDDefSuaBHgcF6eooLBo4cLT\nZ87gTWvUQchdxLi4uPzyyy9cLrd///4KCgoDBw5ssYJAIFBQUHjrdxUUFAQCQfM1c3JyXNz/NQkB\nh/O4qvL9z0GKxS2zKTw8fMH8+eHh4VOmTElOSvr111+dnJw49fVXr1xRVFCorasDkrx86VJISMiU\ngIAhQ4b8uHVraWnpJ598MiUgAABoAF/MmiUWi6OiokQi0bt2oQurJYhGsViZwVDGR7Q6Ern7n3Hn\nzp1Jkyapq6unp6cnJSVdu3at6UcEQRAEkZKS4unp+dbvenp6UrO4UUGjpKTU/KCmjWIfPly+YsXd\n6GiSJH/55ZetP/442c9v2LBhgYGBAVOnPnv2jMPhXAgJ2bFz54njx/l8/p07dzZt2XLj5s2mE6ji\nly99fX11dXXlMF8AYGZmpnV83L7il7IuBP2L3EVMZWVlQ0PDy9eqq6updkNDQxqNFhQU9Msvv1Cz\ntc2ePRsA/vnnn5iYmJiYmH/++ad379737t1buXLlmTNn+vX7gDG8H6yxsbG4uDgvLy8/P7+xsVFd\nXV1PT48qiUanU2c9lRUVNtbWWlpapqam9fX1Tk5OWpqaGurqTY9BGhkbnzp1qq62lsPhSLA2hNpC\n7k6U5s6dCwA//fRTi3YWi3Xs2LGwsLCvv/560qRJ5eXl/fv3BwA+n08tNDQ0GBgY/PXXX8nJybt3\n73ZxcZFgVYmJiYqKipcuXWIwmenp6U2XeGg0WtNBCltZuba2FgA49fWKioq0N04HSJJUUlJSU1Pj\ncDhqamoSLA+hVpO7iJk5c2bzWWJ9fHx27dpFLQ8YMGDAgAHUspGR0bx58wBgxowZzb8+adKkSZMm\nSbyqsLCwdRs2OPbsmZCQEBER4ezsHLRokYWl5YwZMzZv2mRkaAgAenp6jY2NX335paqq6pszsYkI\nYulXX7FYrAaBwNDISOIVSk9iAv3qZbqePjlB1pUgaZC7iPnpp5+aX7JVlvTcrLY2h8Td3/84gqLi\nv1Jg3rx5OtraAODi4mLRrZuWltbL4mI1NTV1dfUDf/zBZDL79uunoKDw07ZtZaWlBoaGCkwmdUds\n3bp11BsemAzGxk2buFyuoaEhvVPdTkpOov2yndHDgZwwU9alICmQu4g5f/78p59++ttvv1EfXVxc\nqGsukqKg2JqHKnV1X93AptPp2jo60Gz+Jn19/abVFBUVTc3MqGVtbW0AoFZuaqEaEeo45C5iCgsL\neTxeVlYW9VGyj1kjKTkSCEc+g3FG8nd7ovOTu4ihLvT+9ddfsbGxhoaGffr0ee9XkMw1KAFHHXhs\naHkJCnV4chcxAFBcXOzq6koNtN+zZ8+iRYtkXRFCXZY8HnimpKSMHDmysbExOjr6zp07si4Hoa5M\nHiMGACwsLJp+lSyCS4hqRO/9D8QtvygSib6YNavFs44VlZXUZaODBw8SH/CS/cjIyKYr2Qh1BHJ3\norRs2bI7d+6kpKRER0dnZWVJdpAuAGQEZFRdf/+zzn1z+7HM/vVqgsTExIexsWlpaY6OjiRJVlVV\nKSkpPU1PT0xMNDQ07N27N51OF4lENTU1mlpaDAajnsNpbGxUVVFRZL3qR0wQly9daj7nCUIyJ3cR\nY21tTaPRBg8eDADOzs7Ozs6yruiVsNu3N2zYcPv2bUdHx4N//PHo0SOr7t2VlZVv3rjRrVu3yMhI\nJyen1atXV1dVGRkZrVmzZuyYMY69etFptD1791I9xNy/37dfv9iHD2W7Ix/LEyr3wEstYAFIeIwS\n6gjkLmKoi7sVFRXU00lvDpOVlaysrG9Wrpw7dy5BELGxsUf//BMA4uPjtbS0xo0bFxkZWV5eTorF\nhw4fXhwUxKmvNzU1/fXXX4MWLaKmlyVJ8tq1a99++22nixgtEDpCLRuUMWK6JHm8FpOVldWzZ88+\nffq4uLh8//33si4HAKC6ujo+Pv7bb7+Ne/y4vr7+rWOOeTyelrY2AKhraAiFQj19fQBgMpnU5Zvi\n4uL4uLgdv/zy6NGj3Nzcdq6/Q5mfk+2ZnBRcXiHrQhCAHB7FAMDz58/9/PxSU1PXrl37119/yboc\nAIB7d+8uWLjw008/NTc3j334kMPhVFdVEWKxoqJifX09tY6urm5uTk5dXV1RYaGKikqLHnR0dLZu\n3crj83Nyc5vGCsun5w0N6TxepahR1oUgAPmMGADQ09PLysrq3r27xKc3Ue6hLKp9/x9uOutfx48F\nhYW+vr76+vqTfX2vX7+++Msvly1b5uLiMm/+/N/27z918qSlpaWamtrw4cO/XLx48uTJymw2Nc+c\nlZUV9aoHFovl3qcPn89/kpj45lyRCMmKPEaMpqZmQUHBoEGDXF1dfX19m9qLi4ufPn2qoqLS/DZT\nbW1tfHy8trY29faG7Ozs/Px8fX39d72MynJba2aYbBr+Z2xiMicwEAD+fP2K8t+bveXTd/Jk38mT\nqeUvv/wSAJYsXdq8Hzab3TTHG0IdgTxGTN++fR0cHBgMRlhY2LBhw5ra/fz86uvrs7Kybt++7eHh\nQTUuW7bs7t27dXV1x44dc3R0dHNzo+aEvXr1qsRveCPU9cjj5V6xWBwYGKinp7d8+fKnT59Sjfn5\n+Tk5OXfv3l22bNmZM2eaVg4ODo6MjFy1alVwcHBycvLAgQOvXr3q5+f35MkT2VSPUKcijxETHh5e\nU1Pz8OHDefPm/fjjj1QjSZIqKiqqqqrOzs5NE5jExsZaWFiYmJj06tWrtLTUyckpLi7O398/JCRk\n9OjRstsDhDoNeYwYkiQHDhzo6Og4640B+wBga2tbXl7etGbzuzMEQdBoNDabzWAwSkpK+Hx+8xfo\nIYTeJHfXYq5cuZKUlLR//35dXd2wsLDmkxZR7t275+TkRC3r6+s/ffqUJEmRSAQAp0+fHjly5OHD\nhxcvXhwXF9e7d2/JvguitrZWQ0PjrT/i8XjvfUFfXV0dNb0JvrgXdRxyFzHHjh17+PAhnU7fsGED\nAIwaNYpqp96qHRUVdeXKFW9v74qKisjIyMmTJ6urq58+ffratWt9+/Y1MzM7e/YsNVWAu7v7W/u/\nWlX1oqHhvWXMNDBQZTCat9TU1trZ2LzIy2Oz2U2NmZmZubm5w4cP37Zt29offmD++ystzP7iC0Mj\nIzs7O+pmE0IdgdxFzLlz5wCgurq6traWRqN169aNatfW1p49e/bs2bPV1dVnzJhRWlp66tSpyZMn\nf/fdd2vWrNHX19+0aZOqqurOnTs/+eQTExOTkSNHvrX/g8XF1z9gytdxOjotIiY6KuqTAQPu3bv3\n2Wef1dTUnD93ztraOis7OyIigq2kNGjQIDqN9jw39+atW6NGjjQ2MQkPCyspKent7t6zZ0+qB0Mj\no3379rXpdwchSZPHazEcDueTTz4ZMmSIo6PjlStXmtq3bt2ak5OTmJhoZGTUs2fPkJAQAAgMDMzJ\nyXnw4IGFhYWurm5cXFxOTk50dHTzV+pKRNSdOxs3bgwPDweA1d9/b2BgIBAKjY2NbW1srLp3v3jx\nokAgWLN2raWl5erVq4VC4Q8//GBgZPTj1q1NF5Pq6uqmTZ16+vRpyRaGUFvIY8Q8ePDA3d09LS3t\nwoULf//9t6zLAQAgSbKsrMzKyqqgoEAkEonF4tFjxgwbNszAwMDC0tLMzAwASkpK7GxtfXx8DI2M\n6urq+vTp4zNsmLaODvn67Q379u07dPhwSEgIj8eT6d4g9H9yd6JEMTc3Z7PZdnZ2si7klZzs7LS0\ntFkzZ2ZkZBQVFr51naZp26hfFRQVAYD2+iMAUM8NmBgbczgciU/eglDryGPE2Nrazpgxo6CgICYm\nZs6cObIuBwAgIjJy95493t7eYbdvR0VFKSgonDl9WonN7tWrV1xcnLe3NwAYGhpmZWVdOH++tLT0\nzaeQCII4duyYtrZ2Xl7em7fJOqAnifTTp+jaOuTsd88B8bA/XAkC5x7tWBaSNHmMGDU1tV9++YXL\n5Xp7e7eY7LHtFhoZjfqAyYy0mP/6ne/Ro0ffvn0BwOuTTxITEydOmnT58mUTY2MLS8vhPj6lpaWT\nJ09WVFTctn17ZETEzz//zGKxqKer/Pz96XQ6ANDpdHNz85KSkt//+IP+xly0HVBWJu3Ab4xuFuTs\nFe9cJ7UnXNAErha05rkv1DHIY8TEx8e/fPly5cqV0uh8eKsmS2ua6JbNZnt6egJAQEAA1TJ6zJim\n1UxMTD6fPp1aplbz8vKiPtJotM8++6y1VSMkLZ3gnzuJ09XVvXPnTtNsbQgh6ZHHiKmqqqJehctm\ns6dNmybrcpC0rMnPm5GVGVFbK+tC5Jo8nih9+umnHA6Hx+MpKCg0H0rbJq9v96CO405tbSqXO0hd\nA97xWAZqB/IYMQ0NDWPGjImIiNDT0zt9+vSgQYPa3ieTyeTx+Sod6VaxQCBgMOXx/y/qUOTxj2B0\ndLRIJLpz587jx49/++03iUQMjUajMRXquB1ozBuDyWAy5PH/L+pQ5PSP4IgRIwYMGODo6PhQclOC\nMBgMxn8+poiQHJLTiNm/f79QKExISMjKytq8efPChQt1dHRkXVTnQAPoBjwAUAc6AE3W5aCOTh4j\nRl1dXU9PLzQ0FABYLFZoaOiMGTMwYj4QA8RH4REAmIEFgFGr++kPlX9CngqwAFozkgh1FvIYMf37\n94+Li5N1FfJOBUTmwGMBzsDdxcnjuJh3WbFihampqYODQ1FRUVPj77//bmpq6ubm9vz5cwA4dOiQ\nqampqalpdna27CpFqNPAiHmlsrLyxIkT586ds7Gxaf6Gh+3bt+/du9fZ2fnQoUNFRUVbtmyJjo6+\ndu2anM+4iNAHwoh5pb6+Xk1NrX///sOHD2+agSAnJ4fH402YMCEgICAhISEpKYkkyY0bN549e5aJ\nQ04Q+gDyGDHFxcXGxsZsNvutDxB4e3s3RUx5ebmdnR2NRmu6G11aWurp6Xnjxo3bt2/jDAQIvZc8\n/lOckpLy+eefb9++/a0/TU9PNzB49QoTOp1eXl7e/MkAHx+fefPmicXiyMjICRMmSHYGAoS6HnmM\nGCaTyefzWzTS6XQul1tXV/fkyRNHR0eCIHg8Xt++fQsKCgoKCqgX+jKZzOLiYoIg8vPzjYxaf78W\nIfkhjxGjqqp64MABaqpZb2/vH374AQDMzMxsbW29vLyeP38eHh6enJy8YMGC2NjYadOmDR48mMvl\nHj9+fODAgZWVlV5eXoWFhWFhYbLeD4Q6AXmMGHNz80uXLlGTojUfcXf27Nns7Gw2m+3q6srlcg8d\nOgQAP//8c2pqqoaGBjWXyOPHjzMyMnR0dDrOe38R6sjkMWKSk5NTUlLefOudvr5+09QlKioq1JyQ\n6urq1PvlKNra2s0/IoT+mzxGDACkpqZSt43U1dXNzc1lXQ5CXZY83rQGgJMnTzo7Ozs7O69Y8e6X\nUyOE2kwej2K8vb2ppwEAAOcb6oDie0PIQrC3lnUdSBLkMWKeP3++f/9+atnV1bWDTKUkJ8Ju0RfN\nV9DVhSvvPnx8ag/nVGCABuCgoy5AHk+U1NXVXV1dXV1dMzIy4uPjZV2OfBEIoKwMystlXQdqL/J4\nFGNiYkIduYwfP37hwoWyLgehrkweI6akpCQqKgoA8vLyZF0Lag/uSU9yGxp+727tj8/Htzt5jJi0\ntLS5c+cCgLGx8YYNG2RdDpI6MUmKcQ4aGZHHazF2dnZ3796tq6tLSkrC174gJFXyGDHp6ek3b94E\nAB6P98cff8i6HIS6Mrk7UYqPj//66695PN7FixdFIhE+aoSQVMldxNja2n777bd5eXmjR48GAFNT\nU1lXJF/YQJhBow7OjiI35O5ESU1Nbfr06VZWVtevX1dSUkpOTpZ1RfLFDWqOwaO9kCjrQlA7kbuI\nAYCMjIy9e/eeOnVKQ0Oj+bWYiIiIVatWbd26VSQSNTWmpKSsWrVq586d9fX1VMvly5c3bdrU3kUj\n1DnJY8QUFBRMmjTJ0NCQzWY3NQoEgtmzZ/N4vAMHDpw5c6apffHixXl5eadOnQoJCQGAhoaGoKCg\nffv2yaBuhDoheYwYAHj06FHTr5SSkhI6nb579+4VK1bExsZSjbW1tQ8fPjx27NiSJUvOnj0LACEh\nIdR7ZBBCH0IeI2bQoEEFBQXR0dGTJ09esmRJUzuDwaDT6Y6OjjExMVTL06dP3d3dWSyWsbFxY2Mj\nQRA//vjj2rVrZVQ4Qp2PPEaMoqJiTEzMpUuXsrOz37xpzWQym1+LaZreBACCg4Pd3d1tbGyojzjJ\nCULvJXcRQxBEcHDwhQsXHBwcZs2atXjx4hYrpKamDhw4kFq2tbV9+PAhQRBlZWV0Ov3p06fBwcEm\nJiYVFRUzZ85ks9k4yQlC/03uImb//v1z5syZPXt27969hULhTz/9RLXr6+vzeLzVq1fv2LHDxcUl\nNzd3+fLl2trabm5u8+bN27dv3+TJk7ds2cLn84uKinR1df/66y/Z7ghCnYLcRUxlZeXZs2cvXrxo\nb28fFhZmYWFBtbPZ7IMHD1ZXV/v6+k6fPl1BQYF6Vfi+fftYLNZnn302ceJEak0lJaWZM2fKqn6E\nOhe5G90LACKRyNzcnM1mFxYWstnspnlORo0aNWrUKGrZzMzs22+/BQB3d3d3d/fmX1dWVn7XTJII\noRbkMWImTZpELXTr1m3SpEnU3WgkVfGP6StXMNlsOBH0znUih8Cu/dBTD3q0Y2FI2uQuYhYuXNh0\nygMAmpqasqtFjtTVQXwcTUXlv9ap0oYMe1BRw4jpUuQuYgwNDQ0NDWVdBULyQu4u9yKE2hNGDEJI\nijBiEEJShBGDEJIiubvci2TCCrhr4KUi0AE0ZViGY2JCiVD4p63dGC0tGZYhVzBiUHvQAuFgKGMA\nQ7YRIyJJEUnifCftCU+UEEJShBGDEJIijBiEkBRhxCCEpAgj5v+qq6tTU1OzsrKaN/J4vNTU1Bcv\nXgCAWCxOTU1NTU2trKyUTYmdTUoy3bOvomdfxaQnOHGSnMI7Sv83ZcqUnJyc0tLSqKgoNzc3qnHV\nqlUXLlwgCOLMmTMEQcyaNUtHR6e+vj45OVlJSUm2BXd8PB6kp9GoBWVZF4NkAo9iXikqKkpOTo6N\njZ0/f/7p06eb2v/++++IiIjFixf//fff1tbWqampcXFxDAaDOq5BbRcxBAJOwUp8A08XhRHzikgk\nUlVV1dHR6d27d0pKCtUYHx+vp6dnY2PTv3//wsJCMzMzNTU1giCEQqFsq+1K+GwoM4BKHVnXgaQD\nI6YlZ2fnly9fUsuNjY1GRkY02r+uI6xfv97Y2NjGxgZnIEDovfBaTEv37t2zt7enlrW1tbOzs5sP\nBr127dquXbsiIyMZDAbOQPBe6iDygloAUIdGWdeCZAMj5hUVFRUOh5Oenh4eHt6rV6/q6uq4uLjP\nPvuMyWRGRET8888/jo6OBQUFM2bM2Lx5c+/evWVdb8dCTTzFYMC/D/jAHLgbIRUAjMC0WBaFIZnD\nE6VXdHV1x48f379//5iYmM8//7ygoGDr1q0AsGTJkgkTJoSFhS1YsODp06dCoXD9+vXa2tqZmZmy\nLrmjKC6m6WqwdDVYZ0+/mtYu7wUtwE8hwE8B71UjPIr5vwMHDhw4cKDpY2RkJAAsX758+fLlVIu1\ntXVdXZ1siutU6urg+lU6AEzpRTOVdTFItjBikMTU1tIOHmAAgJEhPsqMXsGIQa0kFkPeCxoACIWv\nzoYqymH7T0wAWLmKkGVlqCPBiEGtxKmDPj0V4W2BoghiMxAAABs6aNaMTE+vJ0Q/W1j2U1OTdS1d\nHEYMkjxz4B2DRPjPG0kPPeC7o2CqCn7tWdlrqTxunUjEITpoAnYlGDGorbRBeBySAUAPzC+BKgCo\ngei93+KqQF43AHzMq6vDiEFtRQfSGPgAYATc/fAMcBQMagbHxSCEpAgjBiEkRXiihFqJCeQAqAAA\nI/yHCr0bRgxqJRYQ6z/++aNnPWDzetBUhrHSqwx1JBgxqF1V6MDdAaCngBEjL/AQFyEkRRgxqD00\nKkK5LlToyrqON4xOTx+ZlpbK48q6kC4LT5TQxxHT4Yw/AMAIe1D/4G8lO8H0M6DCgINSK6x1HnA4\nBCmuw2G+UoNHMf/3448/urq6Dhw4sKKioqnx1KlTrq6uPj4+hYWFYrF4/vz5rq6uvr6+AoFAhqXK\nkJgOB+fBwXmQ0uv9K5frw/HpcHw6lOlLvzLUIWHEvFJTU7N79+4NGzYwmcy//vqrqX39+vULFizQ\n0ND466+/0tPTQ0JC9u7dm5SURL1NBr1VlQ4cnQ1HZ0OiGxybBcdmQamBrGtCMoInSq/U1taqq6uP\nHTu2qKgoNjaWaszLy6uoqJg3b56pqemvv/7avXt3Dw+PAQMG+Pj45OXlybZgmRMzIN0BAKChO8w5\nCQAQZA/7ewEAzHOEk24AACZGAIWyK/GDiUjS+UkiAJy2te+hzJZ1OV0KRkxLHh4eTe++Ky4udnR0\npNFo1KxsiYmJHh4eTWsKBIKEhITyqmrZFCojIpKEp+kA8EzvZXBgOQAEKL0sKS0HgHTxy5LycgDI\nEr+E8nIAKHj5rwWCTn9aXgEF+Y1M5rOSYigsbFBUzM4vgOKXPCWlXE1NKCnhKCvnqapCWVmtikoh\nWxkqyqvU1FiKilBZWaGhUU+jQ011qaamkASorSkuKKhqbAQOp7CoqIbfANz6/OJiTn098HjPS0q5\ntTXQ0JBTVtZQWQlCYWZFpbC0BESijMoqovglEERadQ1ZkA8kmVJbm/fiBQDElJX9yecDwBANTVUG\no8W+Z2dlujk5tefvdheAEdPSixcvtLW1mz7W1NQ0LWtraz9//rzpo4KCwqSJE9/Vz4MHD5rnkaRk\nZWVpa2vr6Eh+2qH79+97enp+yJplvV9Nlbnndct/L1A9N7V89Xph5euFta8Xtrxe2PF6Yd+7y/jw\n3+F1rxdWvV5Y+nph4euFuZ98Qi1UV1eXlZXZ2dm92U9fN1dlZZzV8uNgxPwfj8draGhITU3t1evV\nlcy+fftmZmaWl5dnZGTo6ur26tXrypUrAEDNaU2n0//jr3qfPn2kEQRMJpPFYkljrtu+fftKo2Dp\n9Syl32FVVVVdXV1NTU2J9yyf5O4F8cQ7bk+SJNm3b18Wi/XkyZOrV69qaGgEBQXFxMRMmzYtNTW1\nrKzswIEDw4YNMzMzc3FxefDgwbNnz0xN3/nq66+//rqkpGTw4MHz5s1re81VVVVBQUEAsH//fmpq\np02bNgmFwp9//vk/avgQ+/bti4mJUVBQWLt2rY2NzeHDh8PDwx0cHNauXfv+L79bXl7eqlWrAEBH\nR2fv3r2VlZUbN26sqKhYsmRJ//7929IzZcOGDcbGxnPnzr1x48Zff/2lra29d+9exhvnNR9l3bp1\n1KwSP/30k4GBwcaNG58/fz5p0iRfX9+2F9ykjUWiToB4t9zc3PPnz9+6dYsgiKqqqoiICIIgysrK\nzp8/Hx0dTa2TlJR0/vz5Bw8e/Ec/Fy5cMDAw+P3335lMZmlp6X+s+YEqKirmzp2rp6eXl5dHEERg\nYGBgYODo0aNnzJjRxp6/+uqr8+fPBwYGjh8/vqKiwsDA4MCBA2pqag8fPmxLt8+fP9+3b9/58+fN\nzMyOHz++a9cuHx+fRYsWubu7t7FggiBiYmLodHpgYKBIJLK3t9+5c6elpeXRo0fb2K23t/f58+fP\nnz9fVVV169YtBweHLVu26OvrczicttfcRNZ//JH0SfCPy7ts2bLl+++/JwjCy8vrzp07kurW3Nyc\nihhdXd2Kior4+HhLS8vGxsa29xwaGjp+/PjTp0/7+/sTBDFnzpyffvqpjX0KhcLKykpvb+99+/Z5\ne3tHRUVxuVwVFZWysrK2dNvY2Dh8+PApU6YEBgampKQ4OjoSBLFjx462p623t/ehQ4euXLkiFAoX\nLlz4+++/EwTRvXv3NqZtC7L+4y8DOC5GipydnWNiYqTUuYuLS0lJSWNjWydyJUny8uXLxsbGTS1e\nXl5tn4guLS3NyMgoOjp63LhxVIuSklKPHj3y8/Pb0u39+/dramqGDh3avNHLy+vZs2dt6RYAunfv\nnpCQMG/evGPHjkm2ZzmHl3ulqL6+nsViSalzoVDYfLLtVvv777/Pnj378OHDlJQUqkUiA5ednJzi\n4+N/+OGH4OBgqoUkSaFQ2MZut27d6u/vX1VVVV9fz+PxqEaJFHz48GEA6N+/f2hoqKGhIdXY0NDQ\n9p7lHEaMVPD5fABITU2VyOXeFpSVlfl8fmpqavfu3RUVFdvSVWpq6ldfffX777/b29tnZ2dTZUdG\nRnp7e7e9TgcHBwcHh9raWjabzefzq6urs7Oz33oz+MMRBPHbb7/V1dXx+fx+/foRBCEWiyMjI93c\n3NpeMAAoKCjA699hkUgUExPTxivfCCNG8ry9vYcPH97Q0JCdne3s7Nz2Djkczq5du2pra3ft2jV7\n9mxfX9+ZM2dWVVWNHDmSRmvTPcHly5erqallZ2dv3749KCho3rx5ixcvvnr16qZNm9rSbWZm5q5d\nu4yMjH799dcDBw5YWVl98803RkZGgwYNauO4kps3bwLA0aNHY2Njv/rqq+PHj8+fPz8kJOTEiRNt\n6VYkEs2cOdPOzu7EiROLFy/u37//+PHj4+PjVVRUbGxs2tIzwoiRPE9Pzz179pSWlp47d04iI7Vo\nNJqiouLKlSsBgE6nr169+ujRo0KhcNGiRW3sefr06YWFhQDAYrFUVFTOnj177969o0ePWltbt6Vb\nTU3Nbt26AcD27dv9/f2pI4KKigo/P8lMmuTu7k5dPDp+/HhoaOjWrVt9fHza0iGdTqcGQ61Zs8bf\n35/JZO7atev58+eLFy+mjmtQq+G4GITajxyOi8E7SgghKcKIQQhJEUYMeosePXqoqqqqqqpeunSp\nxY+2bdsmk5JQJ4WXe9Fb8Pn8jIyMI0eOhIeHjx07Njo6uri42Nvbu7Ky8tSpU927d7exsVFSUrKz\ns7tz506/fv2oJOrZs2dpaWlFRYWNjY2k7iKjzg6PYtA7qaurA8D9+/enTp16/fr1YcOGZWVlVVRU\n3Lx58+TJk9euXQOAzZs3V1dXT5s27fz58/Hx8T4+PqGhoaNHj24aF4fkHEYMertJkyatWLFiyJAh\noaGhgwcP9vLyEovFY8eO7dmz56FDh1qsbGpqevbsWU9PT19f3+DgYA8Pj7Y/2YC6BjxRQm+3cuXK\n0NDQBw8e5OXlNTQ0pKSkDB06tMVIP7FY3CJKtLS02rdM1NFhxKC38/DwsLCwmDBhwrJly5KSkpYt\nW/b06VMA4PP5L168MDMzS09PDw8Px6cE0X/DEyX0FtbW1goKCr1793Z1dXV1dX358uXIkSN//PFH\nGo1Go9EmTJgwZMiQqKiolStX9u7dW0FBgRoNrKioSD1AaGJiIodjzNBb4ehehNqPHCYvHsUghKQI\nIwYhJEUYMQghKcKIQQhJEUYMQkiK/gffcoe9RzBaPwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 21 + } + ] + } + ] +} diff --git a/dopamine/colab/utils.py b/dopamine/colab/utils.py index c501d7ce..a4cb17a5 100644 --- a/dopamine/colab/utils.py +++ b/dopamine/colab/utils.py @@ -277,7 +277,7 @@ def read_experiment(log_path, experiment_path, iteration_number=iteration_number, verbose=verbose) summary = summarize_data(raw_data, summary_keys) - for iteration in range(last_iteration): + for iteration in range(last_iteration + 1): # The row contains all the parameters, the iteration, and finally the # requested values. row_data = (list(parameter_tuple) + [iteration] + diff --git a/dopamine/discrete_domains/atari_lib.py b/dopamine/discrete_domains/atari_lib.py index 1f5ac236..55c9858b 100644 --- a/dopamine/discrete_domains/atari_lib.py +++ b/dopamine/discrete_domains/atari_lib.py @@ -29,12 +29,12 @@ import atari_py +import gin import gym from gym.spaces.box import Box import numpy as np import tensorflow as tf -import gin.tf import cv2 slim = tf.contrib.slim diff --git a/dopamine/discrete_domains/checkpointer.py b/dopamine/discrete_domains/checkpointer.py index 08a478af..9e7dd87a 100644 --- a/dopamine/discrete_domains/checkpointer.py +++ b/dopamine/discrete_domains/checkpointer.py @@ -49,20 +49,28 @@ import os import pickle + +import gin import tensorflow as tf CHECKPOINT_DURATION = 4 -def get_latest_checkpoint_number(base_directory): +@gin.configurable +def get_latest_checkpoint_number(base_directory, override_number=None): """Returns the version number of the latest completed checkpoint. Args: base_directory: str, directory in which to look for checkpoint files. + override_number: None or int, allows the user to manually override + the checkpoint number via a gin-binding. Returns: int, the iteration number of the latest checkpoint, or -1 if none was found. """ + if override_number is not None: + return override_number + glob = os.path.join(base_directory, 'sentinel_checkpoint_complete.*') def extract_iteration(x): return int(x[x.rfind('.') + 1:]) diff --git a/dopamine/discrete_domains/run_experiment.py b/dopamine/discrete_domains/run_experiment.py index a211a2ba..142a510d 100644 --- a/dopamine/discrete_domains/run_experiment.py +++ b/dopamine/discrete_domains/run_experiment.py @@ -183,9 +183,12 @@ def __init__(self, self._summary_writer = tf.summary.FileWriter(self._base_dir) self._environment = create_environment_fn() + config = tf.ConfigProto(allow_soft_placement=True) + # Allocate only subset of the GPU memory as needed which allows for running + # multiple agents/workers on the same GPU. + config.gpu_options.allow_growth = True # Set up a session and initialize variables. - self._sess = tf.Session('', - config=tf.ConfigProto(allow_soft_placement=True)) + self._sess = tf.Session('', config=config) self._agent = create_agent_fn(self._sess, self._environment, summary_writer=self._summary_writer) self._summary_writer.add_graph(graph=tf.get_default_graph()) @@ -231,10 +234,11 @@ def _initialize_checkpointer_and_maybe_resume(self, checkpoint_file_prefix): latest_checkpoint_version) if self._agent.unbundle( self._checkpoint_dir, latest_checkpoint_version, experiment_data): - assert 'logs' in experiment_data - assert 'current_iteration' in experiment_data - self._logger.data = experiment_data['logs'] - self._start_iteration = experiment_data['current_iteration'] + 1 + if experiment_data is not None: + assert 'logs' in experiment_data + assert 'current_iteration' in experiment_data + self._logger.data = experiment_data['logs'] + self._start_iteration = experiment_data['current_iteration'] + 1 tf.logging.info('Reloaded checkpoint and will start from iteration %d', self._start_iteration) diff --git a/dopamine/utils/agent_visualizer.py b/dopamine/utils/agent_visualizer.py new file mode 100644 index 00000000..c7300d6b --- /dev/null +++ b/dopamine/utils/agent_visualizer.py @@ -0,0 +1,126 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Code to visualize different aspects of an agent's behaviour. + +This file defines the class AgentVisualizer, which allows one to combine +a number of Plotter objects into a series of single images, generated during +agent interaction with the environment. +If requested, this class will combine the image files into a movie. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import subprocess + + +import gin +import numpy as np +import pygame +import scipy + + +@gin.configurable +class AgentVisualizer(object): + """Code to visualize an agent's behaviour.""" + + def __init__(self, + record_path, + plotters, + screen_width=160, + screen_height=210, + render_rate=1, + file_types=('png', ''), + filename_format='frame_{:06d}'): + """Constructor for the AgentVisualizer class. + + This class generates a series of images built by a set of Plotters. These + images are then saved to disk. + + It can optionally generate a video by concatenating all the images with + ffmpeg. + + Args: + record_path: str, path where to save files. + plotters: list of `Plotter` objects to draw. + screen_width: int, width of generated images. + screen_height: int, height of generated images. + render_rate: int, frame frequency at which to generate files. + file_types: list of str, specifies the file types to generate. + filename_format: str, format to use for saving files. + """ + self.record_path = record_path + self.plotters = plotters + self.screen_width = screen_width + self.screen_height = screen_height + self.render_rate = render_rate + self.file_types = file_types + self.filename_format = filename_format + self.step = 0 + self.record_frame = np.zeros((self.screen_height, self.screen_width, 3), + dtype=np.uint8) + # This is necessary to avoid a `pygame.error: No available video device` + # error. + os.environ['SDL_VIDEODRIVER'] = 'dummy' + pygame.init() + self.screen = pygame.display.set_mode((self.screen_width, + self.screen_height), + 0, 32) + + def visualize(self): + if self.step % self.render_rate == 0: + self.screen.fill((0, 0, 0)) + for plotter in self.plotters: + self.screen = self.screen.copy() # To avoid locked Surfaces issue. + self.screen.blit(plotter.draw(), (plotter.x, plotter.y)) + self.save_frame() + self.step += 1 + + def save_frame(self): + """Save a frame to disk and generate a video, if enabled.""" + screen_buffer = ( + np.frombuffer(self.screen.get_buffer(), dtype=np.int32) + .reshape(self.screen_height, self.screen_width)) + sb = screen_buffer[:, 0:self.screen_width] + self.record_frame[..., 2] = sb % 256 + self.record_frame[..., 1] = (sb >> 8) % 256 + self.record_frame[..., 0] = (sb >> 16) % 256 + frame_number = self.step // self.render_rate + for file_type in self.file_types: + if not file_type: + continue + filename = ( + self.filename_format.format(frame_number) + '.{}'.format(file_type)) + scipy.misc.imsave(os.path.join(self.record_path, filename), + self.record_frame) + + def generate_video(self, video_file='video.mp4'): + """Generates a video, requires 'png' be in file_types. + + Note that this will issue a `subprocess.call` to `ffmpeg`, so only use this + functionality with trusted paths. + + Args: + video_file: str, name of video file to generate. + """ + if 'png' not in self.file_types: + return + os.chdir(self.record_path) + file_regex = self.filename_format.replace('{:', '%').replace('}', '') + file_regex += '.png' + subprocess.call(['ffmpeg', '-r', '30', '-f', 'image2', '-s', '1920x1080', + '-i', file_regex, '-vcodec', 'libx264', '-crf', '25', + '-pix_fmt', 'yuv420p', video_file]) diff --git a/dopamine/utils/atari_plotter.py b/dopamine/utils/atari_plotter.py new file mode 100644 index 00000000..d3aa0e8c --- /dev/null +++ b/dopamine/utils/atari_plotter.py @@ -0,0 +1,68 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AtariPlotter used for rendering Atari 2600 frames. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from dopamine.utils import plotter +import gin +import numpy as np +import pygame + + +@gin.configurable +class AtariPlotter(plotter.Plotter): + """A Plotter for rendering Atari 2600 frames.""" + + _defaults = { + 'x': 0, + 'y': 0, + 'width': 160, + 'height': 210, + } + + def __init__(self, parameter_dict=None): + """Constructor for AtariPlotter. + + Args: + parameter_dict: None or dict of parameter specifications for + visualization. If an expected parameter is present, its value will + be used, otherwise it will use defaults. + """ + super(AtariPlotter, self).__init__(parameter_dict) + assert 'environment' in self.parameters + self.game_surface = pygame.Surface((self.parameters['width'], + self.parameters['height'])) + + def draw(self): + """Render the Atari 2600 frame. + + Returns: + object to be rendered by AgentVisualizer. + """ + environment = self.parameters['environment'] + numpy_surface = np.frombuffer(self.game_surface.get_buffer(), + dtype=np.int32) + obs = environment.render(mode='rgb_array').astype(np.int32) + obs = np.transpose(obs) + obs = np.swapaxes(obs, 1, 2) + obs = obs[2] | (obs[1] << 8) | (obs[0] << 16) + np.copyto(numpy_surface, obs.ravel()) + return pygame.transform.scale(self.game_surface, + (self.parameters['width'], + self.parameters['height'])) diff --git a/dopamine/utils/bar_plotter.py b/dopamine/utils/bar_plotter.py new file mode 100644 index 00000000..92b8179e --- /dev/null +++ b/dopamine/utils/bar_plotter.py @@ -0,0 +1,109 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BarPlotter used for drawing bar plots. + +Note that a side effect of using this class is to change the font used by +matplotlib. Unless you're planning to use matplotlib elsewhere in your code, +this should be a non-issue. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from dopamine.utils import plotter +import gin +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pygame + + +# You can change this to use your own palette. A site with great examples is: +# https://www.dtelepathy.com/blog/inspiration/24-flat-designs-with-compelling-color-palettes +COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'] + + +@gin.configurable +class BarPlotter(plotter.Plotter): + """A Plotter for generating bar plots.""" + + _defaults = { + 'x': 0, + 'y': 0, + 'width': 213, + 'height': 210, + 'fontsize': 30, + 'bg_color': '#f8f7f2', + 'face_color': '#ffffff', + 'colors': COLORS, + 'max_width': 500, + 'figsize': (12, 9), + 'font': {'family': 'Bitstream Vera Sans', + 'weight': 'regular', + 'size': 26}, + } + + def __init__(self, parameter_dict=None): + """Constructor for BarPlotter. + + This expects a callable 'get_bar_data_fn' in the parameters, which will + return a list of distributions for each of the actions. Typically, this + will be a callback from the agent, which will return some useful information + about its performance. + + Args: + parameter_dict: None or dict of parameter specifications for + visualization. If an expected parameter is present, its value will + be used, otherwise it will use defaults. + """ + super(BarPlotter, self).__init__(parameter_dict) + assert 'get_bar_data_fn' in self.parameters + self.fig = plt.figure(frameon=False, figsize=self.parameters['figsize']) + self.plot = self.fig.add_subplot(111) + self.plot_surface = None + # This sets the font used by the plotter to be the desired font. + matplotlib.rc('font', **self.parameters['font']) + + def draw(self): + """Draw the bar plot. + + If `parameter_dict` contains a 'legend' key pointing to a list of labels, + this will be used as the legend labels in the plot. + + Returns: + object to be rendered by AgentVisualizer. + """ + self._setup_plot() + num_colors = len(self.parameters['colors']) + bar_data = self.parameters['get_bar_data_fn']() + num_actions, num_bins = bar_data.shape + for i in range(num_actions): + self.plot.bar(np.arange(num_bins), bar_data[i], + color=self.parameters['colors'][i % num_colors]) + if 'legend' in self.parameters: + self.plot.legend(self.parameters['legend']) + self.fig.canvas.draw() + # Now transfer to surface. + width, height = self.fig.canvas.get_width_height() + if self.plot_surface is None: + self.plot_surface = pygame.Surface((width, height)) + plot_buffer = np.frombuffer(self.fig.canvas.buffer_rgba(), np.uint32) + surf_buffer = np.frombuffer(self.plot_surface.get_buffer(), + dtype=np.int32) + np.copyto(surf_buffer, plot_buffer) + return pygame.transform.smoothscale( + self.plot_surface, + (self.parameters['width'], self.parameters['height'])) diff --git a/dopamine/utils/example_viz.py b/dopamine/utils/example_viz.py new file mode 100644 index 00000000..2adf7cea --- /dev/null +++ b/dopamine/utils/example_viz.py @@ -0,0 +1,66 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Sample file to generate visualizations. + +To run, point FLAGS.restore_checkpoint to the TensorFlow checkpoint of a +trained agent. As an example, you can download to `/tmp/checkpoints` the files +linked below: + # pylint: disable=line-too-long + * https://storage.cloud.google.com/download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.data-00000-of-00001 + * https://storage.cloud.google.com/download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.index + * https://storage.cloud.google.com/download-dopamine-rl/colab/samples/rainbow/SpaceInvaders_v4/checkpoints/tf_ckpt-199.meta + # pylint: enable=line-too-long + +You can then run the binary with: + +``` +python example_viz.py \ + --agent='rainbow' \ + --game='SpaceInvaders' \ + --num_steps=1000 \ + --root_dir='/tmp/dopamine' \ + --restore_ckpt=/tmp/checkpoints/colab_samples_rainbow_SpaceInvaders_v4_checkpoints_tf_ckpt-199 +``` + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from absl import app +from absl import flags +from dopamine.utils import example_viz_lib + +flags.DEFINE_string('agent', 'dqn', 'Agent to visualize.') +flags.DEFINE_string('game', 'Breakout', 'Game to visualize.') +flags.DEFINE_string('root_dir', '/tmp/dopamine/', 'Root directory.') +flags.DEFINE_string('restore_checkpoint', None, + 'Path to checkpoint to restore for visualizing.') +flags.DEFINE_integer('num_steps', 2000, 'Number of steps to run.') + +FLAGS = flags.FLAGS + + +def main(_): + example_viz_lib.run(agent=FLAGS.agent, + game=FLAGS.game, + num_steps=FLAGS.num_steps, + root_dir=FLAGS.root_dir, + restore_ckpt=FLAGS.restore_checkpoint) + +if __name__ == '__main__': + app.run(main) diff --git a/dopamine/utils/example_viz_lib.py b/dopamine/utils/example_viz_lib.py new file mode 100644 index 00000000..e2c0afd6 --- /dev/null +++ b/dopamine/utils/example_viz_lib.py @@ -0,0 +1,240 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Library used by example_viz.py to generate visualizations. + +This file illustrates the following: + - How to subclass an existing agent to add visualization functionality. + - For DQN we visualize the cumulative rewards and the Q-values for each + action (MyDQNAgent). + - For Rainbow we visualize the cumulative rewards and the Q-value + distributions for each action (MyRainbowAgent). + - How to subclass Runner to run in eval mode, lay out the different subplots, + generate the visualizations, and compile them into a video (MyRunner). + - The function `run()` is the main entrypoint for running everything. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from dopamine.agents.dqn import dqn_agent +from dopamine.agents.rainbow import rainbow_agent +from dopamine.discrete_domains import iteration_statistics +from dopamine.discrete_domains import run_experiment +from dopamine.utils import agent_visualizer +from dopamine.utils import atari_plotter +from dopamine.utils import bar_plotter +from dopamine.utils import line_plotter +import gin +import numpy as np +import tensorflow as tf + +slim = tf.contrib.slim + + +class MyDQNAgent(dqn_agent.DQNAgent): + """Sample DQN agent to visualize Q-values and rewards.""" + + def __init__(self, sess, num_actions, summary_writer=None): + super(MyDQNAgent, self).__init__(sess, num_actions, + summary_writer=summary_writer) + self.q_values = [[] for _ in range(num_actions)] + self.rewards = [] + + def step(self, reward, observation): + self.rewards.append(reward) + return super(MyDQNAgent, self).step(reward, observation) + + def _select_action(self): + action = super(MyDQNAgent, self)._select_action() + q_vals = self._sess.run(self._net_outputs.q_values, + {self.state_ph: self.state})[0] + for i in range(len(q_vals)): + self.q_values[i].append(q_vals[i]) + return action + + def reload_checkpoint(self, checkpoint_path): + global_vars = set([x.name for x in tf.global_variables()]) + ckpt_vars = [ + '{}:0'.format(name) + for name, _ in tf.train.list_variables(checkpoint_path) + ] + include_vars = list(global_vars.intersection(set(ckpt_vars))) + variables_to_restore = slim.get_variables_to_restore(include=include_vars) + if variables_to_restore: + reloader = tf.train.Saver(var_list=variables_to_restore) + reloader.restore(self._sess, checkpoint_path) + tf.logging.info('Done restoring from %s', checkpoint_path) + else: + tf.logging.info('Nothing to restore!') + + def get_q_values(self): + return self.q_values + + def get_rewards(self): + return [np.cumsum(self.rewards)] + + +class MyRainbowAgent(rainbow_agent.RainbowAgent): + """Sample Rainbow agent to visualize Q-values and rewards.""" + + def __init__(self, sess, num_actions, summary_writer=None): + super(MyRainbowAgent, self).__init__(sess, num_actions, + summary_writer=summary_writer) + self.rewards = [] + + def step(self, reward, observation): + self.rewards.append(reward) + return super(MyRainbowAgent, self).step(reward, observation) + + def reload_checkpoint(self, checkpoint_path): + global_vars = set([x.name for x in tf.global_variables()]) + ckpt_vars = [ + '{}:0'.format(name) + for name, _ in tf.train.list_variables(checkpoint_path) + ] + include_vars = list(global_vars.intersection(set(ckpt_vars))) + variables_to_restore = slim.get_variables_to_restore(include=include_vars) + if variables_to_restore: + reloader = tf.train.Saver(var_list=variables_to_restore) + reloader.restore(self._sess, checkpoint_path) + tf.logging.info('Done restoring from %s', checkpoint_path) + else: + tf.logging.info('Nothing to restore!') + + def get_probabilities(self): + return self._sess.run(tf.squeeze(self._net_outputs.probabilities), + {self.state_ph: self.state}) + + def get_rewards(self): + return [np.cumsum(self.rewards)] + + +class MyRunner(run_experiment.Runner): + """Sample Runner class to generate visualizations.""" + + def __init__(self, base_dir, trained_agent_ckpt_path, create_agent_fn): + self._trained_agent_ckpt_path = trained_agent_ckpt_path + super(MyRunner, self).__init__(base_dir, create_agent_fn) + + def _initialize_checkpointer_and_maybe_resume(self, checkpoint_file_prefix): + self._agent.reload_checkpoint(self._trained_agent_ckpt_path) + self._start_iteration = 0 + + def _run_one_iteration(self, iteration): + statistics = iteration_statistics.IterationStatistics() + tf.logging.info('Starting iteration %d', iteration) + _, _ = self._run_eval_phase(statistics) + return statistics.data_lists + + def visualize(self, record_path, num_global_steps=500): + if not tf.gfile.Exists(record_path): + tf.gfile.MakeDirs(record_path) + self._agent.eval_mode = True + + # Set up the game playback rendering. + atari_params = {'environment': self._environment} + atari_plot = atari_plotter.AtariPlotter(parameter_dict=atari_params) + # Plot the rewards received next to it. + reward_params = {'x': atari_plot.parameters['width'], + 'xlabel': 'Timestep', + 'ylabel': 'Reward', + 'title': 'Rewards', + 'get_line_data_fn': self._agent.get_rewards} + reward_plot = line_plotter.LinePlotter(parameter_dict=reward_params) + action_names = [ + 'Action {}'.format(x) for x in range(self._agent.num_actions)] + # Plot Q-values (DQN) or Q-value distributions (Rainbow). + q_params = {'x': atari_plot.parameters['width'] // 2, + 'y': atari_plot.parameters['height'], + 'legend': action_names} + if 'DQN' in self._agent.__class__.__name__: + q_params['xlabel'] = 'Timestep' + q_params['ylabel'] = 'Q-Value' + q_params['title'] = 'Q-Values' + q_params['get_line_data_fn'] = self._agent.get_q_values + q_plot = line_plotter.LinePlotter(parameter_dict=q_params) + else: + q_params['xlabel'] = 'Return' + q_params['ylabel'] = 'Return probability' + q_params['title'] = 'Return distribution' + q_params['get_bar_data_fn'] = self._agent.get_probabilities + q_plot = bar_plotter.BarPlotter(parameter_dict=q_params) + screen_width = ( + atari_plot.parameters['width'] + reward_plot.parameters['width']) + screen_height = ( + atari_plot.parameters['height'] + q_plot.parameters['height']) + # Dimensions need to be divisible by 2: + if screen_width % 2 > 0: + screen_width += 1 + if screen_height % 2 > 0: + screen_height += 1 + visualizer = agent_visualizer.AgentVisualizer( + record_path=record_path, plotters=[atari_plot, reward_plot, q_plot], + screen_width=screen_width, screen_height=screen_height) + global_step = 0 + while global_step < num_global_steps: + initial_observation = self._environment.reset() + action = self._agent.begin_episode(initial_observation) + while True: + observation, reward, is_terminal, _ = self._environment.step(action) + global_step += 1 + visualizer.visualize() + if self._environment.game_over or global_step >= num_global_steps: + break + elif is_terminal: + self._agent.end_episode(reward) + action = self._agent.begin_episode(observation) + else: + action = self._agent.step(reward, observation) + self._end_episode(reward) + visualizer.generate_video() + + +def create_dqn_agent(sess, environment, summary_writer=None): + return MyDQNAgent(sess, num_actions=environment.action_space.n, + summary_writer=summary_writer) + + +def create_rainbow_agent(sess, environment, summary_writer=None): + return MyRainbowAgent(sess, num_actions=environment.action_space.n, + summary_writer=summary_writer) + + +def create_runner(base_dir, trained_agent_ckpt_path, agent='dqn'): + create_agent = create_dqn_agent if agent == 'dqn' else create_rainbow_agent + return MyRunner(base_dir, trained_agent_ckpt_path, create_agent) + + +def run(agent, game, num_steps, root_dir, restore_ckpt): + """Main entrypoint for running and generating visualizations. + + Args: + agent: str, agent type to use. + game: str, Atari 2600 game to run. + num_steps: int, number of steps to play game. + root_dir: str, root directory where files will be stored. + restore_ckpt: str, path to the checkpoint to reload. + """ + config = """ + atari_lib.create_atari_environment.game_name = '{}' + WrappedReplayBuffer.replay_capacity = 300 + """.format(game) + base_dir = os.path.join(root_dir, 'agent_viz', game, agent) + gin.parse_config(config) + runner = create_runner(base_dir, restore_ckpt, agent) + runner.visualize(os.path.join(base_dir, 'images'), num_global_steps=num_steps) diff --git a/dopamine/utils/line_plotter.py b/dopamine/utils/line_plotter.py new file mode 100644 index 00000000..82013242 --- /dev/null +++ b/dopamine/utils/line_plotter.py @@ -0,0 +1,113 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LinePlotter used for drawing line plots. + +Note that a side effect of using this class is to change the font used by +matplotlib. Unless you're planning to use matplotlib elsewhere in your code, +this should be a non-issue. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from dopamine.utils import plotter +import gin +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pygame + + +# You can change this to use your own palette. A site with great examples is: +# https://www.dtelepathy.com/blog/inspiration/24-flat-designs-with-compelling-color-palettes +COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'] + + +@gin.configurable +class LinePlotter(plotter.Plotter): + """A Plotter for generating line plots.""" + + _defaults = { + 'x': 0, + 'y': 0, + 'width': 213, + 'height': 210, + 'fontsize': 30, + 'bg_color': '#f8f7f2', + 'face_color': '#ffffff', + 'colors': COLORS, + 'max_width': 500, + 'figsize': (12, 9), + 'font': {'family': 'Bitstream Vera Sans', + 'weight': 'regular', + 'size': 26}, + 'linewidth': 5, + } + + def __init__(self, parameter_dict=None): + """Constructor for LinePlotter. + + This expects a callable 'get_line_data_fn' in the parameters, which + will return a list of list of floats, each one representing a line + to be drawn. Typically, this will be a callback from the agent, + which will return some useful information about its performance. + + Args: + parameter_dict: None or dict of parameter specifications for + visualization. If an expected parameter is present, its value will + be used, otherwise it will use defaults. + """ + super(LinePlotter, self).__init__(parameter_dict) + assert 'get_line_data_fn' in self.parameters + self.fig = plt.figure(frameon=False, figsize=self.parameters['figsize']) + self.plot = self.fig.add_subplot(111) + self.plot_surface = None + matplotlib.rc('font', **self.parameters['font']) + + def draw(self): + """Draw the line plot. + + If `parameter_dict` contains a 'legend' key pointing to a list of labels, + this will be used as the legend labels in the plot. + + Returns: + object to be rendered by AgentVisualizer. + """ + self._setup_plot() + num_colors = len(self.parameters['colors']) + max_xlim = 0 + line_data = self.parameters['get_line_data_fn']() + for i in range(len(line_data)): + self.plot.plot(line_data[i], + linewidth=self.parameters['linewidth'], + color=self.parameters['colors'][i % num_colors]) + max_xlim = max(max_xlim, len(line_data[i])) + min_xlim = max(0, max_xlim - self.parameters['max_width']) + self.plot.set_xlim(min_xlim, max_xlim) + if 'legend' in self.parameters: + self.plot.legend(self.parameters['legend']) + self.fig.canvas.draw() + # Now transfer to surface. + width, height = self.fig.canvas.get_width_height() + if self.plot_surface is None: + self.plot_surface = pygame.Surface((width, height)) + plot_buffer = np.frombuffer(self.fig.canvas.buffer_rgba(), np.uint32) + surf_buffer = np.frombuffer(self.plot_surface.get_buffer(), + dtype=np.int32) + np.copyto(surf_buffer, plot_buffer) + return pygame.transform.smoothscale( + self.plot_surface, + (self.parameters['width'], self.parameters['height'])) diff --git a/dopamine/utils/plotter.py b/dopamine/utils/plotter.py new file mode 100644 index 00000000..85c3ebbf --- /dev/null +++ b/dopamine/utils/plotter.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2018 The Dopamine Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Base class for plotters. + +This class provides the core functionality for Plotter objects. Specifically, it +initializes `self.parameters` with the values passed through the constructor or +with the provided defaults (specified in each child class), and specifies the +abstract `draw()` method, which child classes will need to implement. + +This class also provides a helper function `_setup_plot` for Plotters based on +matplotlib. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import abc + + +class Plotter(object): + """Abstract base class for plotters.""" + __metaclass__ = abc.ABCMeta + + def __init__(self, parameter_dict=None): + """Constructor for a Plotter, each child class must define _defaults. + + It will ensure there are values for 'x' and 'y' in `self.parameters`. The + other key/values will come from either `parameter_dict` or, if not specified + there, from `self._defaults`. + + Args: + parameter_dict: None or dict of parameter specifications for + visualization. If an expected parameter is present, its value will + be used, otherwise it will use defaults. + """ + self.parameters = {'x': 0, 'y': 0} + self.parameters.update(self._defaults) + self.parameters.update(parameter_dict) + + def _setup_plot(self): + """Helpful common functionality when rendering matplotlib-style plots.""" + self.plot.cla() # Clear current figure. + self.fig.patch.set_facecolor(self.parameters['face_color']) + try: + self.plot.set_facecolor(self.parameters['bg_color']) + except AttributeError: + self.plot.set_axis_bgcolor(self.parameters['bg_color']) + if 'xlabel' in self.parameters: + self.plot.set_xlabel(self.parameters['xlabel'], + fontsize=self.parameters['fontsize'] - 2) + if 'ylabel' in self.parameters: + self.plot.set_ylabel(self.parameters['ylabel'], + fontsize=self.parameters['fontsize'] - 2) + if 'title' in self.parameters: + self.plot.set_title(self.parameters['title'], + fontsize=self.parameters['fontsize'] + 2) + self.plot.tick_params(labelsize=self.parameters['fontsize']) + + @abc.abstractmethod + def draw(self): + """Draw a plot. + + Returns: + object to be rendered by AgentVisualizer. + """ + pass + + @property + def x(self): + return self.parameters['x'] + + @property + def y(self): + return self.parameters['y'] diff --git a/setup.py b/setup.py index 15c17504..ed2d7e44 100644 --- a/setup.py +++ b/setup.py @@ -20,17 +20,12 @@ """ -import codecs from os import path from setuptools import find_packages from setuptools import setup here = path.abspath(path.dirname(__file__)) -# Get the long description from the README file. -with codecs.open(path.join(here, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - install_requires = ['gin-config >= 0.1.1', 'absl-py >= 0.2.2', 'opencv-python >= 3.4.1.15', 'gym >= 0.10.5'] @@ -43,17 +38,16 @@ setup( name='dopamine_rl', - version='2.0.3', + version='2.0.5', include_package_data=True, packages=find_packages(exclude=['docs']), # Required package_data={'testdata': ['testdata/*.gin']}, install_requires=install_requires, tests_require=tests_require, description=dopamine_description, - long_description=long_description, + long_description=dopamine_description, url='https://github.com/google/dopamine', # Optional author='The Dopamine Team', # Optional - author_email='opensource@google.com', classifiers=[ # Optional 'Development Status :: 4 - Beta', diff --git a/tests/dopamine/agents/dqn/dqn_agent_test.py b/tests/dopamine/agents/dqn/dqn_agent_test.py index 6cd03298..18150f05 100644 --- a/tests/dopamine/agents/dqn/dqn_agent_test.py +++ b/tests/dopamine/agents/dqn/dqn_agent_test.py @@ -52,7 +52,7 @@ def setUp(self): self.zero_state = np.zeros( (1,) + self.observation_shape + (self.stack_size,)) - def _create_test_agent(self, sess): + def _create_test_agent(self, sess, allow_partial_reload=False): stack_size = self.stack_size class MockDQNAgent(dqn_agent.DQNAgent): @@ -84,7 +84,8 @@ def _network_template(self, state): epsilon_fn=lambda w, x, y, z: 0.0, # No exploration. update_period=self.update_period, target_update_period=self.target_update_period, - epsilon_eval=0.0) # No exploration during evaluation. + epsilon_eval=0.0, # No exploration during evaluation. + allow_partial_reload=allow_partial_reload) # This ensures non-random action choices (since epsilon_eval = 0.0) and # skips the train_step. agent.eval_mode = True @@ -325,6 +326,25 @@ def testUnbundlingWithFailingReplayBuffer(self): # False. self.assertFalse(agent.unbundle(self._test_subdir, 1729, bundle)) + def testUnbundlingWithNoBundleDictionary(self): + with tf.Session() as sess: + agent = self._create_test_agent(sess) + agent._replay = mock.Mock() + self.assertFalse(agent.unbundle(self._test_subdir, 1729, None)) + + def testPartialUnbundling(self): + with tf.Session() as sess: + agent = self._create_test_agent(sess, allow_partial_reload=True) + # These values don't reflect the actual types of these attributes, but are + # used merely for facility of testing. + agent.state = 'state' + agent.training_steps = 'training_steps' + iteration_number = 1729 + _ = agent.bundle_and_checkpoint(self._test_subdir, iteration_number) + # Both the ReplayBuffer and bundle_dictionary checks will fail, + # but this will be ignored since we're allowing partial reloads. + self.assertTrue(agent.unbundle(self._test_subdir, 1729, None)) + def testBundling(self): with tf.Session() as sess: agent = self._create_test_agent(sess) diff --git a/tests/dopamine/discrete_domains/checkpointer_test.py b/tests/dopamine/discrete_domains/checkpointer_test.py index 03f0e55b..f5c54df6 100644 --- a/tests/dopamine/discrete_domains/checkpointer_test.py +++ b/tests/dopamine/discrete_domains/checkpointer_test.py @@ -85,6 +85,13 @@ def testLoadLatestCheckpointWithEmptyDir(self): self.assertEqual( -1, checkpointer.get_latest_checkpoint_number(self._test_subdir)) + def testLoadLatestCheckpointWithOverride(self): + override_number = 1729 + self.assertEqual( + override_number, + checkpointer.get_latest_checkpoint_number( + '/ignored', override_number=override_number)) + def testLoadLatestCheckpoint(self): exp_checkpointer = checkpointer.Checkpointer(self._test_subdir) first_iter = 1729