policy/02-ActorCritic_CartPole.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "29029eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from typing import Dict, List, Tuple\n",
    "\n",
    "import gymnasium as gym\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from IPython.display import clear_output\n",
    "from torch.distributions import Categorical\n",
    "import math\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "66b7eece",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4,) 4 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.7/site-packages/torch/cuda/__init__.py:88: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 10010). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:109.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"CartPole-v1\", render_mode=\"rgb_array\")\n",
    "\n",
    "device = torch.device(\n",
    "    \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    ")\n",
    "\n",
    "obs_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.n\n",
    "print(env.observation_space.shape, obs_dim, action_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b68dd438",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActionNetwork(nn.Module):\n",
    "    def __init__(self, in_dim: int, out_dim: int):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        super(ActionNetwork, self).__init__()\n",
    "\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(in_dim, 128), \n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128), \n",
    "            nn.ReLU(), \n",
    "            nn.Linear(128, out_dim)\n",
    "        )\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\"\"\"\n",
    "        return F.softmax(self.layers(x))\n",
    "    \n",
    "class QValueNetwork(nn.Module):\n",
    "    def __init__(self, in_dim: int, action_dim: int):\n",
    "        '''状态价值，用来评估动作的好坏程度'''\n",
    "        super(QValueNetwork, self).__init__()\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(in_dim, 128), \n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128), \n",
    "            nn.ReLU(), \n",
    "            nn.Linear(128, action_dim)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\"\"\"\n",
    "        return self.layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fbaec634",
   "metadata": {},
   "outputs": [],
   "source": [
    "actor = ActionNetwork(obs_dim, action_dim)\n",
    "critic = QValueNetwork(obs_dim, action_dim)\n",
    "\n",
    "actor_optimizer = optim.Adam(actor.parameters(), lr=0.0001)\n",
    "critic_optimizer = optim.Adam(critic.parameters(), lr=0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c15cd0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_action(state):\n",
    "    '''策略网络做出决策，给出一个动作，并让智能体执行'''\n",
    "    action_probs = actor(torch.FloatTensor(state).to(device))\n",
    "    m = Categorical(action_probs)\n",
    "    action = m.sample()\n",
    "    loss = m.log_prob(action)\n",
    "    return action.item(), loss\n",
    "\n",
    "def critic_value(state, action):\n",
    "    index_tensor = torch.tensor([action])\n",
    "    return critic(torch.FloatTensor(state).to(device)).gather(0, index_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9c98c3bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:16: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "gamma = 0.98\n",
    "\n",
    "steps = []\n",
    "U_s = []\n",
    "view_losses = []\n",
    "max_epoch = 2000\n",
    "for i in range(max_epoch):\n",
    "    score = 0\n",
    "    step = 0\n",
    "    \n",
    "    state, _ = env.reset(seed=3)\n",
    "    trajectories = []\n",
    "    while True:\n",
    "        action, log_prob = select_action(state)\n",
    "        '''从环境中观测到奖励和新的状态'''\n",
    "        next_state, reward, terminated, truncated, _ = env.step(action)\n",
    "        done = terminated or truncated\n",
    "        \n",
    "        '''根据策略网络做决策,但不让智能体执行动作'''\n",
    "        next_action, next_loss = select_action(next_state)\n",
    "        \n",
    "        step += 1\n",
    "        \n",
    "        trajectories.append([state, action, reward, next_state, next_action, log_prob, done])\n",
    "        if done or step > 200:\n",
    "            steps.append(step)\n",
    "            break\n",
    "        \n",
    "        state = next_state\n",
    "        \n",
    "    # 结束一轮游戏\n",
    "#     U_t = []\n",
    "#     R = 0\n",
    "#     for state, action, reward, next_state, next_action, log_prob, done in trajectories[::-1]:\n",
    "#         R = reward + gamma * R\n",
    "#         U_t.insert(0, R)\n",
    "        \n",
    "#         '''让价值网络打分'''\n",
    "#         q_t = critic_value(state, action)\n",
    "        \n",
    "    for state, action, reward, next_state, next_action, log_prob, done in trajectories[::-1]:        \n",
    "        mask = 1 - done\n",
    "        \n",
    "        '''让价值网络打分'''\n",
    "        q_t = critic_value(state, action)\n",
    "        next_q_t = critic_value(next_state, next_action)\n",
    "        '''计算价值网络的 TD 目标和 TD 误差'''\n",
    "        y_t = reward + gamma * next_q_t * mask\n",
    "        state_loss = F.smooth_l1_loss(q_t, y_t)\n",
    "        '''更新价值网络'''\n",
    "        critic_optimizer.zero_grad()\n",
    "        state_loss.backward()\n",
    "        critic_optimizer.step()\n",
    "        \n",
    "    loss_sum = torch.tensor([0.0])\n",
    "    for state, action, reward, next_state, next_action, log_prob, done in trajectories[::-1]:    \n",
    "        q_t = critic_value(state, action)\n",
    "        loss_sum += -q_t.item() * log_prob\n",
    "    \n",
    "    '''更新策略网络'''\n",
    "    actor_optimizer.zero_grad()\n",
    "    loss_sum.backward()\n",
    "    actor_optimizer.step()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b77c7e3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAE/CAYAAABW/Dj8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnXd8VFXax39PQugllNBLAJFiARERGyqgIthd7HX1RV1dV3f3VSy7oq+6uDbW1bVgwYLYsIPSZEVESugdAoQaktBSSE/O+8fcSe7M3Jnb6zzfzyefzJy5957ntt997nPOeQ4JIcAwDMP4nxS3DWAYhmGsgQWdYRgmILCgMwzDBAQWdIZhmIDAgs4wDBMQWNAZhmECAgs6wzBMQGBBZ5IKIppIRB+5bQfD2AELOsMwTEBgQWcCCxE9TET7iKiYiLYQ0VgAjwK4lohKiGiNtFwrInqHiHKl5Z8molTpt9uI6Fci+jcRFRLRZiIaKavjNiLaIdWxk4hudGdvGQZo4LYBDGMHRNQXwH0AThNC7CeiTACpAJ4FcJwQ4ibZ4u8DyANwHIBmAL4HsAfAm9LvpwP4AkA7AFcB+JKIegKoAPCKVMcWIuoEoI3Nu8YwcWEPnQkqNQAaARhARGlCiBwhxPbohYioA4CLATwghDgmhMgH8DKA62SL5QOYLISoEkJ8CmALgLHSb7UATiSiJkKIXCHEBjt3imESwYLOBBIhRDaABwBMBJBPRJ8QUWeFRXsASAOQS0RHiegoQp55e9ky+0RkFrtdADoLIY4BuBbA3dL6M4monw27wzCaYEFnAosQ4mMhxNkIibYA8Jz0X84ehEIn7YQQ6dJfSyHECbJluhARyb53B7BfqmO2EOICAJ0AbAYwxabdYRhVWNCZQEJEfYloBBE1AlAOoAyhMEwegEwiSgEAIUQugDkAXiSilkSUQkS9iehc2ebaA7ifiNKIaByA/gBmEVEHIrqMiJoh9FAokepgGFdgQWeCSiMAkwAcBHAAIVF+FMDn0u+HiGil9PkWAA0BbARwBKEG0E6ybS0F0Efa1jMAfieEOITQ/fMXhLz1wwDOBfAH+3aJYRJDPMEFw8SHiG4DcKcUumEYT8MeOsMwTEBgQWcYhgkIHHJhGIYJCOyhMwzDBAQWdIZhmIDgiVwu7dq1E5mZmW6bwTAM40lWrFhxUAiRobacJwQ9MzMTWVlZbpvBMAzjSYhol5blOOTCMAwTEFjQGYZhAgILOsMwTEBgQWcYhgkILOgMwzABgQWdYRgmILCgMwzDBAQWdIZhmICgKuhE1I2IFhDRJiLaQER/ksrbENFcItom/W8tlRMRvUJE2US0logG270TDMMwjDYPvRrAX4QQ/QEMA3AvEQ0AMAHAfCFEHwDzpe9AaAb1PtLfeACvW241wzCWs3j7QVRW17pthm2UVdZg2c7DprYhP0Z7Dpdie0FJ3W/Z+cXYf7TM1PbNoiroQohcIcRK6XMxgE0AugC4HMD70mLvA7hC+nw5gA9EiCUA0omoExiG8Szr9xXihilL8Y8fNrltim088uVaXPPmb9hzuNTQ+mv3HsUNU5biuR83AwDO+ecCjHzx57rfR720EGdO+skSW42iK4ZORJkATkFojsUO0gS74Yl220uLdUFoJvUwe6UyhmE8yqFjlQCA7PwSlSX9y6bcYgDAscpqQ+v74RhpFnQiag5gBoAHhBBFiRZVKIuZRYOIxhNRFhFlFRQUaDWDYXzBK/O34Y2ft7ttBmMDXp4SSJOgE1EaQmI+TQjxpVScFw6lSP/zpfK9ALrJVu+K0KzoEQgh3hJCDBFCDMnIUM0KyTC+4qW5WzHph82G1998oAi/Zh+00CLGLEqeqtfQ0suFALwDYJMQ4iXZT98CuFX6fCuAb2Tlt0i9XYYBKAyHZhiG0cboyb/gxreXum1GoBCe9q2tQUs+9LMA3AxgHRGtlsoeBTAJwGdEdAeA3QDGSb/NAjAGQDaAUgC3W2oxwzAMo4iqoAshFiH+28ZIheUFgHtN2sUwDGMpZFHQJCRx3oRHijIMw2ggFH32NizoDMMkBWZj6F72zMOwoDMM44seHFZhVejFi7CgMwyTBP0/6jHqqXPIhWECTG2twF0fZpnOD8I4Q5A98zAs6AxjkCOllZi9IQ93f7TCbVMYBgALOsOYxg+NZWoE33dNjoFFLOgMYxA/xFSZWIyGXvxwtlnQGcYkQfD7grAPWjHffdEiQ2yABZ1hmKSAG0UZhlElCDLhtX0oKK7A1rxiS7eZDDF0Lcm5GIZJQPBlwnmG/3MByqpqkDNprOXbNhxD99pTTwH20BnGID64v31LWVWN2yb4EhZ0hjGJlxvJmFiCHHphQWcYhgkILOgMwyQVZnu7eNnDZ0FnGIP4oZGMsQ4/dHtkQWcYkwRh6H+YAO1KUsKCzjAMExBY0BmGqcPKMNKOghLkFZVbt0GPIARwsKTCbTMU4YFFDGMQP8RU3WTEiz8DgC2Dg9xA/rA7/4X/umZHIthDZxiTBCnszDF0bRSXV7ttgiIs6AxjlAA56NxjJxioCjoRvUtE+US0Xlb2KRGtlv5yiGi1VJ5JRGWy396w03iGYayBPfNgoCWGPhXAqwA+CBcIIa4NfyaiFwEUypbfLoQYZJWBDMM4B3vq6nj54acq6EKIhUSUqfQbhaZsuQbACGvNYhgf4eEbnEkuzMbQzwGQJ4TYJivrSUSriOhnIjrH5PYZxrME0Zt1yvssr6rBSU/Mxo/rDzhTYZJgVtCvBzBd9j0XQHchxCkA/gzgYyJqqbQiEY0noiwiyiooKDBpBsNoY8HmfMzflGfJtrz86q0Xpx9OuYXlKK6oxqQfNjlWp9nz5Yfnt2FBJ6IGAK4C8Gm4TAhRIYQ4JH1eAWA7gOOV1hdCvCWEGCKEGJKRkWHUDIbRxe1Tl+OO97PcNsNzOP1wsjpdwuYDRSgsqzK9neU5h1Fbm9i2oCbnGgVgsxBib7iAiDKIKFX63AtAHwA7zJnIMN7Gu7e3fpz21MmiCkdP/gXXvvmbSl2Jt7FgSz7GvfEb3lucY4lNbqCl2+J0AL8B6EtEe4noDumn6xAZbgGA4QDWEtEaAF8AuFsIcdhKgxmGsQ8/h5E2HzA3B+n+o2UAgOx8a+cydRItvVyuj1N+m0LZDAAzzJvFMP4hCNkWg9jAG43aaUqVDkJtbZwFfHCMeKQowxjF/zpuKRv3F2HV7iNum6FKvIdXivTDmr1HE67v5ec3J+diGMYSkRrzyi8AtCXjUqruWEU1BIDmjeyVpXj7GhZ6s6EbN2FBZxjGNeTO8glPzAZgX3ZGtbBSisa4k5fDUxxyYRiTePgNXDNeFimrkHvmb/68HWP+9UvE76kp2g4Ch1wYJoB4uT8yEx8i4B8/bI4pT1ERdD/kv2cPnWEYx/Gil6vmoB8trXTGEBOwoDMMU4fjbx0ecnrVYuj3TFvpkCXGYUFnGJN40dtk4hPvfGkMoXs60MaCzjBMHX6IE9uF1l4uXoYFnWEMwp65Gdw7eGoDi/wMCzrDMHU4HUP3koSmBEANA7ALDOMuQei+mMyhljCaPXQPn24WdCYpWbHrME56YrajXdFue28Znvsxtv+zFwjCQ8ksHHJhGJ/y6k/ZKK6oxkoLkklpjaX/d0sBXv/vdtP12YlTnroX2x+MCvq/52/zTMZNHinKMAbxxi1sLY7H0D3kFRuNob84dytOzWxtrTEGYQ+dYRjXYuhe8WwBcyGXGpVp65yCBZ1JajykJ4zNqJ1qrYKu9BbjleuIBZ0JDBO/3YAVu5yf8dAj97IpjIRapi/bjY+X7jZVr6dCLiZMOeKRPC8s6ExgmLo4B1e/nnii4GjM6ImXwgVu8MiX6/DoV+sMrevGkVM71WauBa80drOgMwzjWgzdO/65ObzybGdBZ5Iar9yIjP2onWqt14KXrxnutsgwZvHwDe4GV/7nV1xycme3zYiL2beCrF2xYxe80hTAgs4EAqPxbK/ciEFi1e6jWLX7aMJl3PRy7ajaK167asiFiN4lonwiWi8rm0hE+4hotfQ3RvbbI0SUTURbiOgiuwxnGLfxyD3sCOv3FaKqptby7Tr5QE2GZ7eWGPpUAKMVyl8WQgyS/mYBABENAHAdgBOkdf5DRKlWGcswVuMVz8rLZOeX4JJ/L8JzCvNwBgkzl4JX3vRUBV0IsRCA1s69lwP4RAhRIYTYCSAbwFAT9jGM5wl6YquDJRUAgLX7Cl22xBx2niWvOAZmerncR0RrpZBMOJFBFwB7ZMvslcoYxlaM3lBe8ay8QsLjaKFoufkQDPIpNyrorwPoDWAQgFwAL0rlSsdK8cwR0XgiyiKirIKCAoNmMIx7eMUrc5KsnMPYfajUsu3Z0f+9tlbgm9X74uZXiXfazJxPrzgGhgRdCJEnhKgRQtQCmIL6sMpeAN1ki3YFsD/ONt4SQgwRQgzJyMgwYgbDmCYZRTkRasL0uzd+w/DnF0SUlVZW22iRfj5fsQd/+mQ1pi7OiSj3iObaiiFBJ6JOsq9XAgj3gPkWwHVE1IiIegLoA2CZORMZxtsE/aGgJoQD/j7bETu0crCkUvpfEVGeDDF01X7oRDQdwHkA2hHRXgBPADiPiAYhdIxyANwFAEKIDUT0GYCNAKoB3CuEqLHHdIapx+j95JVXZa+QSJisjHs7IYDx6rDjlCtdRw9+uhrtmjfEY2MH2FCjMqqCLoS4XqH4nQTLPwPgGTNGMYyf8IhzZgq3HmxeeqCaSbamtOpXq/YBgKOCzrlcmKTGjKcYpO6KboUM3KjXiSrXy7p45hWVO1BjCBZ0JinxUh5uL6F0WPx2rMLmuvXA3bi/CJf8e1Hd99Ofne9Y3SzoTFKS7LnMjWDHIXPjWWF3DD2v2DmPPBoWdCYQuJmcK0gPh9zCcjz0xRpUVluft0WO1w7ZpB82Y/WexAnFEuGV/eFsi0xSY+pG9MhNbAXhB9uOgmPYUXAMI/t3wEUndHTXKIMYGaz0xs/emHHILOyhM0mJ3+LCdhP9YJMfnWQ9VDW1AsXlVZqW9coxYkFnkpIghUkYe3jyuw04aeIcVFSrD6XxyuXEgs4EAjMDi16euxX3frzS8br9hpX7ae0gpTjbMlnFVytD/cgrbG5PsBIWdEYzS3ccwoLN+W6bYSlCAP+avw0z1+a6bYqrJAoZ2BlN8EPoS4v37ZXdYEFnNHPtW0tw+9TlbpthCVqFpLSyGtsLShR/SxbPPExQw1Rq+7Vhv3oe+ILiCtVlnIAFnUlKtIrT/3yQhZEv/myzNe4TJK3W6y1/tGRXwt9vmLIUC7YkfjM9dKxSX6U2wYLOBAK7Jrj4NfuQbXX/zwdZOP3ZecZW9jlOPECW7jyMzAkzkZ2v/IYVZtOBYtVt7T9aZpVZtsKCziQ1eoWloLgCD3+xFhXVNabbE+ZuzENekTde1RPG0G2MD1ux6XjnMDxQ6Lcd6g/laB7/eh2KK+rzvDduoH1qZDfD6SzoTFJitDHu6Zkb8WnWHsxal4sJX66z2Cr9HCqpwKJtB902I3B8tGR3xPfGadoF3c3oFQs641mEECjSOLDDyLbNLG/H1GlGuGHKUtz0zlLUxpluzSh29z4pLtc/y5EQ2gf6yAnvSbxzruVSSPHG6VaFBZ3xLJ8u34OTJ85Bdr56jNNov2avdDczypa80LGxej+UxM+qR0ZuYRmun7JE93pv/7ITJ02cY0M8W33P7pmmfZwCh1wYRoGfpBh1dv4x2+rQG0PXsvyd7zvftdPeRkZrJSpf1m6g50H044YDANQbKONtMv5bh8+f6jJY0JmkRG9IQY9gztvk/8FXdoZc5PFoKx5E5jcRnD6bLOiMD/DODefVEI13jpA6jdPMyY7efa2b8CJIne3jwILOJCVGb26vaYKTDxir9j1FZrQrE1x49KFsBZwPnfEB6neg00LrNVEIPaDcM8qM9/v87M14bYH2fORm99JrD2UrYQ+dSUp0x9CjvwdYFIyg53h8H5UILVrMP1kW2Qc8pq6YurVVHu+cB+lcsqAzPsA7d5zXPPMwdh4hq/f5uR83J/zd7IAtr54jJ1AVdCJ6l4jyiWi9rOx5ItpMRGuJ6CsiSpfKM4mojIhWS39v2Gk8wxglMDF06b/VdtXU1qKssgYV1TWokvKBJ6rCTD563evoXj60RjI0imqJoU8F8CqAD2RlcwE8IoSoJqLnADwC4GHpt+1CiEGWWskkOd5zuYLmBUbvzh+mrUStANJSCVU16kLoJ7GMPnc+Ml0VVQ9dCLEQwOGosjlCiPDY3SUAutpgG8NIWH/H6e+HLqK+W2mNeczOABS9djiTgBYxdxo1i6LTMgRZwKOxIob+ewA/yL73JKJVRPQzEZ1jwfaZJCVoXrAdeGXGH7s1Mr+oHCt2HTFVt1eOlZ2Y6rZIRI8BqAYwTSrKBdBdCHGIiE4F8DURnSCEKFJYdzyA8QDQvXt3M2YwAcVOT8psiMBr2uC212k4H73GcNr/fLhCto4x/BQWMophD52IbgVwCYAbhXSkhBAVQohD0ucVALYDOF5pfSHEW0KIIUKIIRkZGUbNYBgA7gua3zHdt9tmH/1oaf2MQHyq42NI0IloNEKNoJcJIUpl5RlElCp97gWgD4AdVhjKJB+2TqzA/dD1Y8NOaz0Ndr4QWf0wcjO0o6Xb4nQAvwHoS0R7iegOhHq9tAAwN6p74nAAa4loDYAvANwthDisuGGGYUxjlXSYlbREWr88x7wERKQL0Llu9PJ2P4vdDO2oxtCFENcrFL8TZ9kZAGaYNYph7IZj6M4x7o3fkDNprOJva/cWqq6/cGsBjlXWT4ih9rak99x4+djphXO5MIHA7hhu9E1fY/EMQW7jsedTHbmFZbjl3WWmtlFdK1BRXWORRep4OuTCMEHE7E3358/WWGSJNdj9QAvVkeA3m6o/VmFeiB//ej36Pv6jBdZ4HxZ0xvME6ZXYarwS+lF7oEyetxWPf60/R4sfuxq6aTMLOsM4jBACb/9ibecvr+ve5Hnb8NGSxFkUlajRsGNa3068foysgAWd8Txe8EKtDGn8su0gnp65ybLteQG7xNLKtoodB0Nz00bbarXpHENnGJP4yfuyo0HVid1PdIztqt/u87pgi//nf5XDgs54nvBNvWTHIWROmIk9h0sTr+AhtuUVI3PCTKzde7SurGED7bedWjxW69B5OfM35SFzwkwUFFfoXjcedsWNa21W9O35JbZu32lY0Bnf8MWKvQCA37YfctkS7YQ9wO/W7K8ra5Bi/Su5HkGdujgHALAxNybFkmbWaeg/bgVB6x5qNyzojOcJhyTrJnNwIZtHWC/11pyaErrFqmXClKbDQ3cMnc+YS19dFPHdrjOiRc/lz7J1ewuRW1hukzXehwcWMZ4nfMMmamvyqh+XKtks9zQbpuoJuWhrFDa9/1qEM8FCdkVGlEIuVdW1qKkVSFV404l+0Cgh3w8h/NX+ooYHXQWGCREvPuzmDag3WJIqibdc0BukeqDbjk9QCrnc8PZS3Pj2Ehes8T4s6IxnifYIwwma/ORQhePlcmEy0pCphtJDbmteMS58+WcUllVZXl+sAfZsNl6j6JIdxhN+XfHar4bX9Tos6IxvCIcerOz58OXKfbqW1x9DDxldbbBxT3WtBM+GyfO2YmteCRZtO6hekZawTsJuizYpug2bLa+qlW3e+grmb8qzfJta4Rg641liPVll1SkqrzLcBW/mulxD62lFyUO3BYXNByI2bHN0yo5j9MFvu6zfqEZY0BnfEX0Tnvf8f3H4WKXywhajO4Zu0kN3DJPm+fnh4UavKbvgkAvjCxZszsf0ZaFcING3n1NirlS3Gql1Hrqx13yt/cuVtunkCHSte5Q5YaY9GzZDcPScPXTGH4QHwxiltLIaz84ynj/FqAda15Brk2g42V/Gz154PATg3WTwBmAPnXGEqppaLNhsUd4MA8ry/uJdhrL9uY3WPTUttiZFzbaUsU6IbYAeVCzojCNMnrcVt09djsXZGnpcqGDk/rOqZ4xRfamqkYVcbBAQpU066VG7qYlBfHMwCgs64wg5B0MJtQ5ZEO924wYOx6iNVj1vkz1Z/SyLkydpo6hf7Y4HCzrjO7w8i43VtmndnJePiSkCult2wYLOeIIVu45g8XZt4Rgj97iXBM9pUzR58WZj6D5V3loh8OUqfYPLvAz3cmEcIXzDxxOXq19fDADImTTWKZMcwakOFE7I6f7CMucN0JSYzHjlP6y3d2CZ07CHzvgOIx6uWa9Ye+jDXD0x21MRKzvywsTjaKkDOWGisflJdayixt4KHEaToBPRu0SUT0TrZWVtiGguEW2T/reWyomIXiGibCJaS0SD7TKeSR5EnM9G+cmqLpQq7Dsa69XaEZ5wO6I09Nn5tmzXn4Ec99DqoU8FMDqqbAKA+UKIPgDmS98B4GIAfaS/8QBeN28mExSMeJTRN7Vd8fAf1uXim9XWxlOX7TSeFZBh9KJJ0IUQCwFEX5mXA3hf+vw+gCtk5R+IEEsApBNRJyuMZfyLWQ02G1jQUv0901biT5+sNlmT/noTrh9nAyUV1Viyo34qPiWvX+2Y+2GApN1vHl5qLLcCM42iHYQQuQAghMglovZSeRcAe2TL7ZXKIlofiGg8Qh48unfvbsIMJugQokIurvRDN7ackq1W2P+n6asw32TYSG6GV+fu1BKeCpgmm8KORlGlB3/MIRdCvCWEGCKEGJKRkWGDGUxQ8Pf9ao/1m6IneDZZzc3vLDO3AZtgsdaHGUHPC4dSpP9hd2EvgG6y5boC2A8mqdEyLygA3D99Fb5atTdmWflqRhoV3RIGrwvSk99twNu/7HDbjLjM3nDA1u17/PToxoygfwvgVunzrQC+kZXfIvV2GQagMByaYRg1vl2zHw9+ugZApBi6HXLRSnRM1srZlSLqUfmulfd+zcHTM41nobSbaUv9l1DNTTTF0IloOoDzALQjor0AngAwCcBnRHQHgN0AxkmLzwIwBkA2gFIAt1tsM5Nk7DsS2fXPDT13q/EsXrV6zIn3UuSHRlFGH5oEXQhxfZyfRiosKwDca8YoJrjoEZFwyOWZWZswuHt6XbmXPfRonDLV0GAr681whaDshxXwSFHGF8hH9BmKobt02yt1HrHigeTX3Clew0/OgRZY0BlHMCJAP6xXbhDTexOu2HXE9IxHRrstKjF/s/ZZ4bUeNyPHd95G92anZ+yBk3MxjmI0f7eZvN/hxF9uoBR7nzxvmwuWhFi5+wgKy0I5WT5c4t7s9Iw9sKAzvsArr8ZeGVkYbYbiAKYor10Igav+497DzS7MnJOgha445ML4Dq+IqhLRppnttujhXQ0EQTu+LOiMI5hOXyvzpFwZ+l83MEpf7McuW430Qw+aeDGxsKAzmiivsipvtPnez17VpfKqmpi4tG2C7tWD4DBfrtyLgyXG56kN2nHkGDqjCTcb8oDItLtu3oTxwj1CCLw8dyveXGjtMHrNvWuCpkwayC8ux58/W4PeGc3cNsUzsIfOaOJoqXEvCLAgjaxn/fJ6lGb0sc9u7dsNR4m8fwT1UVUT2qP8ogqXLfEOLOiMbl5bkG14XTPdD8O4Iu4qMXQhlPfNbFZarZ53eLFb3l2G8R9kxfxeWV2L3o/OMmeMRymuqDa8btDebFjQGd08P3uLq/V7MeQSfwXr639xzpa4ceOFWwswR2HAUFG5C/OB2kzQxNgKWNAZ3+FcfhR9NSl56Fa/Tew4eAz//kn/G1KDlOCl4mI9j4UFPckQQqDWhdlprLz5nPLMZqyMnF+0tja+PIfKY0VTi6mJ9if6l3jnTq2elAAKOhMLC3qSccOUpejlYizVqKzIBcuIh2qEPYdL6+uHQO/HZmHmWn+m9mc5VyZoTj4LepLxm2xi4WRgcfZBy7aVyAsWQsQJuZjbrlbUQjtBEy6rCFrYhgWdcQh77hy1yY3/R6HHh1b0WqzkBWvxjBPVEzTBsRJOQxwLCzrjKHqHzoeJd9t9vMy+Kcoi5jFVue+PVdQo9kO3opumFhSTc4nEvzPBg0eKMr6mqMy+7nh6xPi8FxbgiNLAIs2NohzldoOgPejYQ2cYC1ASc60k1BSNgqO6WMCECwAWWdA+ErTDwoLOOEI8T+jdRTuxbm+hbfUaDfEAkflj7AztWMW7i3Yqlj/34xYUlARvePyUX6zNmxMEOOTCOEq0vD71/UZX7NCC/FlgdHTsit1HVJdJ2HsmepKKuNsQcY/lzoPHMGHGWlU7/AYHqWJhD53xNWqDjMzc9FYIhlMxWrVqKmtqHbHDUazICxSwmAsLOuMIprMtBu3Ok2FF1zm17plBPHzsocdiWNCJqC8RrZb9FRHRA0Q0kYj2ycrHWGkw4392FJTghilLUFqpPUte/FCDNTYp4VSXwykJcqhr3b8dBccsssY/mGkfqSdYTzrDMXQhxBYAgwCAiFIB7APwFYDbAbwshHjBEguZwPHsrM1YvP0QFm2zbhRnXAze8wdLKvDdGmeG+b8wZytO7dEGrZqkYUDnlhG/rdtXiCYNU3FaZhtTdQRtAA3gr5CYU1jVKDoSwHYhxC5rnppM0AiHTAxfHgZuvNpageJyY7myx3+QhS15xYbWNcL1U5YAAHImjY0ov+XdZQCAjU9dhF2HSlVHxsYjaMIFOPcG5SesEvTrAEyXfb+PiG4BkAXgL0II9aZ+hkmAET16/efthuvLLSw3vK4dXDR5IfYcLkO/ji00rxNADWdUMN0oSkQNAVwG4HOp6HUAvREKx+QCeDHOeuOJKIuIsgoKCsyawdiMHd7Q+A9XaF427lyeCdZxeyIOK9lzuAwAsPmAtreGnIPHMFc20UUQPXQrqHXowGROmInPs/bYXo8VvVwuBrBSCJEHAEKIPCFEjRCiFsAUAEOVVhJCvCWEGCKEGJKRkWGBGYxdfPBbDtbts2bwj9EHA+uRPmZvOOC2CbZDPuvn4sSzwwpBvx6ycAsRdZL9diWA9RbUwdhIdn4x8orihxj+/s0GrN9XZKoOPwlyflG5ayGXvUdK1RcygJ+Ov1acbOOwAidi/qYEnYiaArgAwJey4n8S0ToiWgvgfAAPmqmDsZ9RLy3E6c/Od6Quo16VkyEfpGf8AAAgAElEQVSDoQ4dCyXOfm6Ba3UnI04+6FIcUHRTjaJCiFIAbaPKbjZlkQU88uU6TF+2O6bHAOMfdh6M7Fcd3e1u8P/NRasmaTHLMSGitSPIA7P8QooDwzgDmctlug8SKSUbevVkfVTMPnr9w8cqcfiY8sz3TCxaG1OTDaUc9nbhhIfOQ/8ZR/l5K/doYpITJ8bosKAzjjJ1cY6h9ThiwPidFK83ijKMXfAoQHP4rUtfMsAhF0YzBwrLkTlhJlZpyL9tBT+uP4DjH/8BZZU1jtTHjXr6eGbWJrdNYKJgQWc0E56O68Mluxyp7/nZm1FZXau537ReOY72MPd7bCg+w+iFQy6MZwk38KzafRTlVepe+p7D+gbMBDE7IJPcsIfOaMbpkER2fgkA4KEZa/HQF+rTm3F/cSbZ4X7ojC9IlOclc8JMDD8+OleP+sOHG/WYoMHdFhnNWHGxLNiSb4ElsSzkvucMwyEXRjtWhFxuf2+5oe0KIbDvaBl+2pynuiwAzNuk/uAwc+3vKChxZjYkhtGBE42iHHJhEqLlOSEAXDx5IYrKqz2RP2fEiz+7bQLDxJDKHjqjFbvic1r9/iKDU70xTLLAMXRGM3b1ctG73cwJM/HN6n222MIwfob7oTOuo0XOozX//77faLpe7uPCBI0UBxSdBZ1JiBYHPbewzH5DGMajtG6apmk59tAZ3Vjdf1vLiM2qGuvDPTzUn/ELWmPjHEM3STj+W1ZZg8wJM/Hmz9tdtsh+rB4y71ZOrKIy5yYeYJgwr90wWPc6tRpvEu6HbhFHSkMz2xjNxW0lHy7ZhSkLdzhWX02twP3TV2HdXuXRnD+u9+bs8MmcPpczS7pH4zT7JJFDLiYJ3xdeuj3+9vV6R1Ob7j9ahm/X7Mc901Yo/n73R8rlYdzSlmQe+l9ZU+u2CUmLmiPRt0OLmDKt9wh76CaJPs61QqCguMIVW/yKW1kPk9lDZ/yF1jcqFnSLySuqwGnPzMPBkuCKutWeba2XXm8YxmbUtPmM3m1j19G4bSeyLQZa0OM9OYMyW/zBkgoc//gPEbMUWd8oamR75h8qL83danobDGM1D4/uF1OW0aKRpnV94aETUQ4RrSOi1USUJZW1IaK5RLRN+t/avKlMNIu3H0JldS3eXrQz5refNudhweb6JFh7j5QZyknODrrzTJ63zW0TmDikKrRsTrvzdE3r+qlR9HwhxCAhxBDp+wQA84UQfQDMl747TliMoo9jUMKzYe9Zvj/hkMvvp2bh9qnLI14hr3nzNwN1GLIMxeVV2HygyMjKSc/r/w1+91q/ouRkd2rVBJ+MH6ZhXR946HG4HMD70uf3AVxhUz2aiNakoHmd8gslUcjlWEVkAi1NecoNHqxb312G0ZN/MbYyw7iEmgMTT5KH9WqLD34/NOG6vgi5IHTLzyGiFUQ0XirrIITIBQDpf3sL6tFvWNCU22L2H1Ufsm8sJk9YufuogfUYxtuYEWUn0udakQ/9LCHEfiJqD2AuEW3WspIk/uMBoHv37haYEUs8MQpOyCX0XynkUvdd9rW0Un0y53h1MAyTuDutml470RXXtIcuhNgv/c8H8BWAoQDyiKgTAEj/Y6aoEUK8JYQYIoQYkpERPecko4XwA8vIhaJ1hCjrOcPUkygOrtZl2PPZFomoGRG1CH8GcCGA9QC+BXCrtNitAL4xU49RvOxdWjG8W8lDj7dMNHd/tEJxkFVFdQ3KKmtQXF4lre/hg8gwHkLNsfLDFHQdAHwlPbUaAPhYCPEjES0H8BkR3QFgN4BxJusxRbQoeWEU4psLd+Duc3ub2kadoBvcoSqFIeaPzFiHRdkHkV9cgZxJY9lDZxiLcKJR1JSgCyF2ABioUH4IwEgz2w46Czbnmxd06b+Vl8mXq+pnG9qaV4wLX15o4dYZv9K/U0tsyuVuqGbwRQydiWXd3kJ8sWIvAGDP4VK8/Yvx7Io7Dx7D1F9jBw4BsjcPmy6UaUt22bNhxnf0ad/cbRMcwcwbqdIbrxy/dFt0nbyicuw9UhpT7lT4t7SyOsJ7ufTVRfjr52sAADe9sxRPz9wUk25Aq2nj3liMid9tREV1/B4qBHI8NJIoHuiFkBbD2MWATi0VyyurEwu6E90WAyHopz87H2c/tyCmPH4famsP7B+mrcTF//oF5VWxoltcHhrME5MEX6MCF5VVx/2tLuRi03WSKDafyNvgdlQmyLx3+2mK5Wppj9lD18kMKcwRTazAWKs4y3ceBgBUK6QmDJ/CIU/Pi5hkQu+AHSWRlA/9N3KpsO4yWrFDi5zo9WEH8RwWtZALcbZFfXyyfHfEd72e4p7Dpfjj9FUJwxtmWLitfqi9mm2Hj1XiD9NW1D31w8vnF5Xj3mkrUVpZLevlEinO78WJuUejdj8ZHUQR5PTEyUr06R7asw0mXzvI8PZSCGjYwHvyY6abrlrIhT10i4g9R8oH9m/frMd3a/bj1+yDttih53y+tiAbs9bVD/4Jh2xemLMFM9fl4tvV+2W9XCI3/OR3GzXVMUtlcNHynMNxf0vmGYWCzDl92mlarmFqCkYN6GC4HiLS7HB1b9NU83a12m+GDi2V0+WqC7od1kTVYX8VzhF9gRyVJhrWGt6o7zRiz5GXb1fNouh9qRUCReVVdWEdAUR46PEsTrTv2fklCW1Yvy9BNzXW80DyxKUDIr4/NqY/AGBoz8iJHRJdc1Zz0QnaHhxnH9cO951/nCV1HpegVw8RoV3zhjHlFeyh28tZk35CTa3Q7AnULWbhcZefwxkrlWP8yrZEGl0rgJMnzsGXK0P9xIWIHPrvdDyc9TyoRJ7ZYb3aImfSWHRr0ySiXAjzcXWt63dv20zTcrVCoCbBzd4kLVVbhVD3ps/sHfsm0CsjsZ3cD10nNULg7Od+iiirFbE+6tWvL1bsG66UX9xK5B5xvFjdPR+twDMzN8Y+hGI6ydQ/qKYv24OHvliruL3PsvYYNTchat4Io84p3dPdNkEzSm+tZt9ktTpag7unY/YDwzVtr2FqfEn78YFz1Lch/Vfzpp8fd3JM2Yh+HRLWwR66TkorarD3SGxK2GjxLCyrwtMzN8XdjtGh9NH1fLZ8Dw6WKE93F+9a/mH9AUz5ZWfMtqK7PWq9GV5bwJMleJVR/Y3HoK1GzbuMhsiYx/nVH87UvQ6B0LdjC9XlBARO7dEaf7tkgOLvrZqk6a47Ho0aKHv7/Toq91EHuB+6JehptDY78DK6qodmKHvN8rq0bitG0BWWAXhQj5/w1LmKcz2GbYy21WhnkAGdQ4JHCtuMh9blQmEgwh1n91T8XUu2QztPiRPn24p86J5BqQFQ6OjxbTQdbdijt3JATWyjqMoCiYsZJiFGBqk5EULQg9qlr8dDtmPf/DwFnSsoD77RLnKme7noeRtQ+T02xKLNQ2f007eD+uu8F2jXvCE2PHmRrXVovfKNhFw2PTXa3u6uKjeE0gTP8TbhsWeVZgIl6PGxTvoKy6rwx+mrUFhapVCLgfiOjK9W1feCiQ25aNqEby9EN2ne2NiLaiOTA2P0iltqCqFZI3deqqMtFUJ/eKJJQ+W485s3n2rIpmjU7j8tgm418fK+2EWgBF3pdGr10FfsOoLF2w8BAPYdjU30Fea9X3fiuzX78Y7CaEw94Y41sjQAYR78dE3cbSk1ivLkE9Zg9D43O9JR78PXiUY1AMhoERo4M+akjgkFSSmE0CQtFW2axfbRTsSZvdsm/F3rbis5PXL0HD8tt9bESwfgvduU87qEMTP4ygjBEnSFs6A1hn7164vrPj88Yx2KyqsUczMkOtHWymtsiEVORXWNYn05B49ZakUycPtZyo1oajj9PL1vRB/btq107zxx6Qn1DYkKWqgkj0TA3y7pH78ehXYqtXi11jeZ287MjPvbCZ1bamoU1XNObzurJ87v1966DVpAsARdqUxHDF3OyRPn4L6PV+qr38KTF+OhR7kfz85Snov7mIGJoJOdi0/saGi9mAyaOtHjb98/sg9uON3aydSVhtTrabgLLxrdBqH3sKhVqaW7Yc6ksbh0YGd9FevAaJdHp9+hAyXoSkcv1Hho7LDO3pAX8X3uxjz8a/42PdUnJNEDICbbrsYYOqMfo70PTAu6y+0d3913tuZlFQcWEWHBX8/DW7fUx8Ctvib7d2qJjq0aW7tRDUTvxsL/PR+/ThihfzsO36OBEnSlG0wIgc+Wax9yn4iHvqiPcSvdi3pPXqKYX+zQf6VwUiw8TZhzqMVs1dDTwGmH9reQNQYb3ZWe7ZqhR9tmeHRMP13raQ2j9NY54EmNF8fFzJiJGxXefKKdrVZN09AlvUnMckq0blrvzRt1Jo3ia0GvqRVYv6++cTHnkMKsRQDe1ZhOVonCsirslOLSJRXxJ5sI1aXv5FXXxh8+Hy0W8R5WjDFaWNBb5JXrjKePBYCLTtAe6rHDm5dvU2lQHcVZVombhvVI+PvArq0AhIbmn9ilJSbLjl0icU90hfdqpy724XBQ2P6rT+0as0yHlvVvAGf0aouurZvg/pHG2ytGn9ip7vOwXm3RrU0TfP9H7W9DZvC1oE+etxWX/HtRwmXMat6l/16E81/4LwCgqkbn8E4VEui5ai8XxhzRAnWhgd4Io0/sZGgoe5h2zZXTsCphR/9tpVCT2QdHPKcm3PBMRPj+j+dgzEmdVNeRfozLT389z4iJsVXI6mjVNA2LHh6BEzu3Mrw9+THs26EFfnloBE7sYnx7evC1oK/bF9v1L5qBT84xVcfuw/G7MEajV3Kra2sx+P/m4s73sxS2Fbk1pclQWOONE93jwaiQmQ27aKV9nBzcTqE6GYrKEkrHN3qdFo0bYH3UwCmzIYvO6erxd6U6GqVZI43tWzob//e1oFvls0RP4KwFpVmBHvpiLfboeADU1obqnrcpL+Y3LR660/G5IBHdVc64B+zMObh2SDdH6gHq7ys9exY+nLpyJ8lq+OyuMzD3wXPRvFED/Pev5+Guc3vp3p4SSqGTXx46P9IOhTo6tGwcd+5QrVw/1LlzFsawoBNRNyJaQESbiGgDEf1JKp9IRPuIaLX0N8Y6c+3hhTlbdK9TVB4bT/95awH+8UP8LI7RJIqhF5ZFjkStUXAF2UM3TnSXZDs89IHdrEuPK3+j+N+L+lq23TBycf3wjtNxx9k90b6F9reCRg1ScNe5vTDjHmMhqKE929T1Zsls1wwDu4aOndlr/OSu6bjljB547YbBdWXd2jRFZtv6Lpvxqji/b3v867pB+OfVsalytTDARNjGKGY89GoAfxFC9AcwDMC9RBTOW/myEGKQ9DfLtJVxsCrZjZbGRaWh/mHkjaUtG2vvr5ooGf9Pm/Mjl1USdM01MdFYde1Ejw+QM6xXG0vqiOZei2blUSKcqvZvlwyIOEZqx4uI8MjF/XXFihO9FQ2SHobXnlbv5d5zXm/N2w6TmkJ46vIT0UPjJBnRXD6oC645TZ+n7WZvVMOCLoTIFUKslD4XA9gEoItVhmnBqgOXqHEyzMCnlGPx0fNu6hEKJZGOx6sLsmPK3vjZf7nOjTQ+WkHOpLEYLetVkkKRc0MqnbaHR/dDzqSxWJCg8S3RGfTTvKt2vO1deUoXjD051PipdF8kChl2Tm+CnEljI0ZiPjy6nz3Xj12vui68QlsSQyeiTACnAFgqFd1HRGuJ6F0iam1FHcr1WrMdoz1ILnp5IaYv3R1Vqn1bT2mczBkIDWqK5miCtwav8sr1pzhe57LHRgKIFJBUIsx54NyYeKqc8PKJcoBEXzta5780y9nHmZsMOevxURHf06W+06mpyvtqero5c6vX0ULHG3Ai5CM/nWrYdgLTgk5EzQHMAPCAEKIIwOsAegMYBCAXwItx1htPRFlElFVQUGDWDFMYPaFb8orx7Zr9huv9Yf2Bus+fLIt+MASTxjrmdbSCds0boX2LUGxWrr1EhFZN09BNGv6u5E3LJ+FW2m70NoFIobCz0Tp6jk85WrpDRi/z9i2nYeKlA+IOnlHKa6SJBIfAyBvME5cpz0akl7duGaI40bPfMSXoRJSGkJhPE0J8CQBCiDwhRI0QohbAFABDldYVQrwlhBgihBiSkZFh1AKD64UoqajGjoISUwN0ouPgeUUVhrYz4ct1hm1g4iMX4/tH9qn7Xl6lPeeNUlKn8ECZhG93Nnp+iaqNl4Pq9RsHx5QN7JaOF8YNRMdWjXFbgiRlZVKOoLQ4HrwaVr1Nt2ychtvOzMRNwxLntbnutG64a3ivuL93aNkYt56RCcD6B29djx9Lt6oNw8PlKBQUewfAJiHES7LyTkKIXOnrlQDWmzMxkQ3m1r/5naVYtfsorhhkPKlP9I0V3ZjJuIv8EjmxSytkPzMGvR+dhYtP6hh/QYnwKEOlkEu4SOntLrNtU8VRy1Zy5nHt8Mly5QnA9eT9/ubeszQtVyoJepO0VFTVJB4xLceOt5SJl52guswkDT1TjHS11IKbbSdmPPSzANwMYERUF8V/EtE6IloL4HwAD1phqBJmD9uq3UcBAF+vNh42YbxN9L2amkJYO/FCPHnZiQnXe/WGU+pyWSvrY6gwnA+lPsssYfaDw7HpqdF1dcu7GUZ36+tkMPFUogyRcnM/u+uM+nITN0zYQ9eSf6Z101AoQ55Uy4sNxHVTR1q83fB+h4+Dkxj20IUQi6CsqbZ1U2SYePz4wDkYPfmXmHKlcJpS19LoC7mjbISfUsglXDS4e2u8cdNg5BaW48nvNoKofkb4cN3yMEX0yMVPx5+B4c8vUN6pBCSSR7m9SpNw9Ouof8q90sqQVx5v1iE55/XNwH9uHIxR/TvgT5+s0l2XG8z787m6wnCJuGt4L/Ro2xRjZekNnMLfI0W999BnXKJfR+WZdbR2Df37pQMw7tSuOEmhH7XSBAzystEndlJs7NXyKp/ezFivjUTdY+W2CSEw9uROkoiHyrsp5EFXY9yQbrhqcBc8OOp4TbaNOalTxMPEi/eqPORyXPvmluVbaZCagktO7uzIpNDR+FvQPfgax3gLra/T7Vs0xvPjBip6tEohl+h7VUm8L5Y8tOHH1zf6R7+GKz0s2uqcwi0aub21AnjthsH48YHhpkS1WaMGeOmaQXXdG7Vid1fs353a1XBD7XnHh/q4X+hQV1Mn8Legs54nBfIQgTx39S1nJE7ZClgjKEohl3jXnrz81B6tkTNpbMTbQ+O0VORMGoucSWMBKPca6dJaPe+21pCLEmaOSbgL6Eld9Xmzdt2qL4wbiG3PGMsuMqBzS+RMGovB3W0bKuM4LOiMYzx75UmG1ntsbOQclU2k8Eb49I9P0D3N6NRhcpS8aL2v08seHYklj4yMKW/UIBVzHxxe911poNOMe86IKUtUfaS99erdSHr7aN7I+FiAvh1bYOb9Z+MvF6iHXkK1x3966PX2GXXMZ/l3kZO7pmPWugPqCzKW8fjY/nh6pvYEZEBoIoIdB4+hU5xUpikUf3DX1YO7ormsZwUR8P39Z2P5zsPYfKAYQGQDppz2LRrhSxP5ysModluM+q7WPS9RGtU+sjk55fHt128cjPziCpzaIzYnTOIYuswumVnD+2TgodF9cePp6m82iThBR9KpRIOzPrvrDPx3awGaNvS1DHkKX3voo3XM+MJYwxWn6E/X88I1A9ElvQlO7RF6tb0uKtmRUtw6zA2nd0ffji3qlrn1jEz0zmiO64YmHlhy07DuWPbYKF2TSMQjWozaNW+Iu8+NTBR1Qf8OaNusYd1EDno5oXNLPP+7UN/pCRf3Q+dWjXFu3wzcmmAm+3jIQwjyx0xKCuEP5x1nyVuLVu4bcRzaNW+IYb3axvzWrU1T3Kwy0xGjD18Lup4BFIx5BndPR1pq4kvmgVF9kDNpLPq0by5brzV+nTCirrtg9KAPeeP23y6JHNrdOb0xmjZsgK1PXxzarsybDYvEyVHx3IHd0vH0FcbCO0pEC3rW4xfE9Iho37IxVvztAhzfQX+XQACYef85GCflPD+zdzssfmSkZs+1gew+GNw9HaP6e6eR7+Su6ch6/AKku9AnOxlhQWfiEp3X45PxZ6BhAkHv1Kox7h8RmlBAz4wv4dPYOC0FV0pvAGf2bos1f78QnVrFbyAcfWJHrP77BRiSGRWSsLhrRTjk0rNdM2x86iKVpZ2lR9umWP/kRXUjPru2bopRAzrgOOmByjnzkwtfB69Y0IHP7z4D4974zZZtN5UNIplxz5lo2CAF1VFJmr6+9yzsPVKK+z5ehd4Zzet6WIRDHZOvVZ9IuUFqCoAafP/Hc9CmWUO8d9tpOLlrK7TS0GjmhOfXIDUFU28/DSd2aeWpeO8Hvx+Kfp1aoHFaKgZ2S8cbN52Kc/qEsjC2kY4LTySeXPjaQ1fqfZBsnBbtnWpAy+AQIDL+Go5/Rz9EB3VLx0UndMS4U7viH1fVhznCOhJPlKcqTO8Vzn53fr/2aGsg9v3omH6614nmpWsG4pohXesmWAhzXt/2lsTjrWT48Rl13QiB0BtL9NB8lvPkwteC3iDJPXSjM+LI14uOWUfTrnkjjJBNMqDUuyItNQXPjxsY0UMjnIUw3hk6r2/7mDKzzuTQnqGYupnN9GjbDP/83UDprcG/3ChlI+yd0VxlSSZI+PqqbWBwhFhQ+GR8bP9kLYSHww/r1QZ3nN2zbpALgIi5Fk/plo6sx0fh3dv0T5YbHkIv9yDjYdWLVnJfDZFcPqgLciaNRYaOeUEZ/+OdgKABWjROQ7+OLer6IycrH91xOm56Z6n6ghLhOLdS97Vv7jsbh0oqUF5Vi97tE8/DuHjCiLi//fmC4zHmpE4Y0Fk5xwoQmkmoqkbg+reW4GhplWlhDz/gm2vICMgwQcT3V/6Azi2x+UAxuqQ3wb6jZYrLjOrfAfM2RU7h1q55IxwsiT8ZxYldWmL0CR3x6oJslFdpm62lfYtGyC82NsFFmJO6tMK6fYURZSd3bYW1ewvjrAGcdVxbPD62P3plNMNT321EzqFS9GjbFK9ePxgb9hfilO6tUV1bi5fnbsP44b1wWmZr/O9FfXGDrC/3WzefipZN0tBK+kvES9cMxPEdWqBznNltgFBDolqyo7D3Pu3O0zF/U57pBs4BnVpiwsX9cNVgR6e2ZRjP4OuQCwDceXYvpDdNw9eyRP0dWjaqm5wAQEQMOMylAxOntuzQojHuG9EHIxP06ZXH8J+58kQse2wU/nzB8RjSw3huiK/+cCZayDzM9i0aYdqdp9cNbw93JQwPQgFCce07z+mFEf064MpTugIALhvYGSd1bYXrhoYG5pzQuRXevnUIhvZsAyLCvecfh9ayJFAXntBRcfCHElcN7mpZZjogNMAk0Ww5WiEi3H1ub01hHoYJIr4X9AGdW2L13y+MiBW+d9tQzH5weN2ExP07tajLYxHmdKkB7fO7z4iIIU+783QAwIj+oYfAmBNDwv97SXCm3DKkLrfGv647pS7RUng49f0j++CLe2KHm189OCS04QZJeUa98Gi5+X85Fw1SU7DuyYsku1ti2WOj0KJxGh4d0x85k8bi1wkjkDNpbN0glGgG90iX/gcn4RDDMNogL/RTHTJkiMjKyjK9nfKqGhQUV0T0tjh8rBJtmjVERXUN8osqkNGiEaprBZo3alD3GxCakeVwaSW6pDfB4WOVaN00ra5Hx6GSCrRp1hD7jpahS3oTEBEOlVQk7FpXUlGNkvJqpDdNw8GSCnRs2RhlVTVo1CAVVTW1SE0hlFfVoLi8Gl1bh+qUb6+kohppqVQ3WYIe5PvFMIz/IaIVQoghqssFSdAZhmGCiFZB933IhWEYhgnBgs4wDBMQWNAZhmECAgs6wzBMQGBBZxiGCQi2CToRjSaiLUSUTUQT7KqHYRiGCWGLoBNRKoDXAFwMYACA64kocVo/hmEYxhR2eehDAWQLIXYIISoBfALgcpvqYhiGYWCfoHcBsEf2fa9UxjAMw9iEXYKulAg1YkgqEY0noiwiyiooKLDJDIZhmOTBrvS5ewHIs0d1BbBfvoAQ4i0AbwEAERUQ0S6DdbUDcNDgunbiRbvYJu140S62SRtetAkwZ1cPLQvZksuFiBoA2ApgJIB9AJYDuEEIscGGurK05DhwGi/axTZpx4t2sU3a8KJNgDN22eKhCyGqieg+ALMBpAJ41w4xZxiGYeqxbcYiIcQsALPs2j7DMAwTSRBGir7ltgFx8KJdbJN2vGgX26QNL9oEOGCXJ/KhMwzDMOYJgofOMAzDwOeC7la+GCLqRkQLiGgTEW0goj9J5ROJaB8RrZb+xsjWeUSycwsRXWSTXTlEtE6qO0sqa0NEc4lom/S/tVRORPSKZNNaIhpsk019ZcdjNREVEdEDTh8rInqXiPKJaL2sTPexIaJbpeW3EdGtNtj0PBFtlur9iojSpfJMIiqTHa83ZOucKp33bMlupXEgZu3Sfb6svD/j2PSpzJ4cIlotlTtyrBLogHvXlRDCl38I9Z7ZDqAXgIYA1gAY4FDdnQAMlj63QKiL5gAAEwH8VWH5AZJ9jQD0lOxOtcGuHADtosr+CWCC9HkCgOekz2MA/IDQILBhAJY6dM4OINSn1tFjBWA4gMEA1hs9NgDaANgh/W8tfW5tsU0XAmggfX5OZlOmfLmo7SwDcIZk7w8ALrbhWOk6X1bfn0o2Rf3+IoC/O3msEuiAa9eVnz101/LFCCFyhRArpc/FADYhcWqDywF8IoSoEELsBJCNkP1OcDmA96XP7wO4Qlb+gQixBEA6EXWy2ZaRALYLIRINIrPlWAkhFgI4rFCXnmNzEYC5QojDQogjAOYCGG2lTUKIOUKIaunrEoQG5cVFsqulEOI3EVKHD2T7YZldCYh3viy9PxPZJHnZ1wCYnmgbVh+rBDrg2nXlZ0H3RL4YIsoEcAqApVLRfdLr1LvhVy04Z6sAMIeIVhDReKmsgxAiFwhdgADaO2yTnOsQedO5ecGvgtMAAAK/SURBVKwA/cfG6WP2e4Q8ujA9iWgVEf1MROfIbN3rkE16zpeTx+ocAHlCiG2yMkePVZQOuHZd+VnQVfPF2G4AUXMAMwA8IIQoAvA6gN4ABgHIReg1EHDO1rOEEIMRSlt8LxENT7Cso8ePiBoCuAzA51KR28cqEfFscMw2InoMQDWAaVJRLoDuQohTAPwZwMdE1NJBm/SeLyfP4/WIdBQcPVYKOhB30Tj1W2aXnwVdNV+MnRBRGkIncZoQ4ksAEELkCSFqhBC1AKagPlTgiK1CiP3S/3wAX0n154VDKdL/fCdtknExgJVCiDzJRlePlYTeY+OIbVKj2CUAbpRCA5BCGoekzysQik8fL9kkD8vYdW3pPV9OHasGAK4C8KnMVseOlZIOwMXrys+CvhxAHyLqKXl/1wH41omKpZjdOwA2CSFekpXLY9BXAgi3yH8L4DoiakREPQH0QahxxkqbmhFRi/BnhBrX1kt1h1vNbwXwjcymW6SW92EACsOviTYR4UW5eaxk6D02swFcSEStpZDDhVKZZRDRaAAPA7hMCFEqK8+g0MQxIKJeCB2XHZJdxUQ0TLoub5Hth5V26T1fTt2fowBsFkLUhVKcOlbxdABuXldGW3i98IdQq/FWhJ7AjzlY79kIvRKtBbBa+hsD4EMA66TybwF0kq3zmGTnFpjshRDHpl4I9SRYA2BD+HgAaAtgPoBt0v82UjkhNKvUdsnmITYer6YADgFoJStz9Fgh9DDJBVCFkEd0h5Fjg1BcO1v6u90Gm7IRiqeGr6s3pGWvls7rGgArAVwq284QhAR2O4BXIQ0YtNgu3efLyvtTySapfCqAu6OWdeRYIb4OuHZd8UhRhmGYgODnkAvDMAwjgwWdYRgmILCgMwzDBAQWdIZhmIDAgs4wDBMQWNAZhmECAgs6wzBMQGBBZxiGCQj/D4Zd966U8dLmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 观察曲线\n",
    "plt.figure(figsize=(20, 5))\n",
    "# plt.subplot(131)\n",
    "# plt.title('frame %s. score: %s' % (frame_idx, np.mean(scores[-10:])))\n",
    "# plt.title('score')\n",
    "# plt.plot(scores)\n",
    "plt.subplot(132)\n",
    "plt.title('steps')\n",
    "plt.plot(steps)\n",
    "# plt.subplot(133)\n",
    "# plt.title('epsilons')\n",
    "# plt.plot(epsilons)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1076ae71",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}