B_Q1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc2f42cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nihang/.local/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time\n",
    "from torchvision import datasets, transforms\n",
    "from torch import nn, optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9c093684",
   "metadata": {},
   "outputs": [],
   "source": [
    "##transform.ToTensor: convert image to numbers\n",
    "##transform.Normalize: normalize the tensor with mean and std.\n",
    "transform = transforms.Compose([transforms.ToTensor(),\n",
    "                              transforms.Normalize((0.5,), (0.5,)),\n",
    "                              ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5a41db58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# download MNIST dataset\n",
    "trainset = datasets.MNIST('./', download=True, train=True, transform=transform)\n",
    "valset = datasets.MNIST('./', download=True, train=False, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)\n",
    "valloader = torch.utils.data.DataLoader(valset, batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "08d3a453",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 1, 28, 28]) torch.Size([64])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f2f93f1af40>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbXElEQVR4nO3df2xV9f3H8dcF6RW0vV2t7W1HwYI/QIEuY9J1KF+UhrYzRqRL8Mcf4AxEV8ywQ12NijqXOliY0TBc5kY1ERQWgcgSMqy2zK1gQAlhmw1t6oDQlkHGvaVIYfTz/aPx6oXy41zu7bv38nwkJ6H33s+9b89u+tzpPT31OeecAAAYYEOsBwAAXJ4IEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMHGF9QBn6u3t1cGDB5Weni6fz2c9DgDAI+ecurq6lJ+fryFDzn2cM+gCdPDgQRUUFFiPAQC4RPv379fIkSPPef+gC1B6erqkvsEzMjKMpwEAeBUOh1VQUBD5fn4uCQvQihUrtGzZMnV0dKioqEivvfaapkyZcsF1X/3YLSMjgwABQBK70McoCTkJ4d1331V1dbWWLFmiTz/9VEVFRSorK9OhQ4cS8XIAgCSUkAAtX75c8+fP10MPPaSbb75Zr7/+ukaMGKE//vGPiXg5AEASinuATp48qZ07d6q0tPTrFxkyRKWlpWpqajrr8T09PQqHw1EbACD1xT1Ahw8f1unTp5Wbmxt1e25urjo6Os56fG1trQKBQGTjDDgAuDyY/yJqTU2NQqFQZNu/f7/1SACAARD3s+Cys7M1dOhQdXZ2Rt3e2dmpYDB41uP9fr/8fn+8xwAADHJxPwJKS0vT5MmTVV9fH7mtt7dX9fX1KikpiffLAQCSVEJ+D6i6ulpz587V9773PU2ZMkWvvPKKuru79dBDDyXi5QAASSghAZozZ47+85//6LnnnlNHR4e+853vaPPmzWedmAAAuHz5nHPOeohvCofDCgQCCoVCXAkBAJLQxX4fNz8LDgBweSJAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACbiHqDnn39ePp8vahs3bly8XwYAkOSuSMST3nLLLfrggw++fpErEvIyAIAklpAyXHHFFQoGg4l4agBAikjIZ0B79+5Vfn6+xowZowcffFD79u0752N7enoUDoejNgBA6ot7gIqLi1VXV6fNmzdr5cqVamtr0+23366urq5+H19bW6tAIBDZCgoK4j0SAGAQ8jnnXCJf4OjRoxo9erSWL1+uhx9++Kz7e3p61NPTE/k6HA6roKBAoVBIGRkZiRwNAJAA4XBYgUDggt/HE352QGZmpm688Ua1tLT0e7/f75ff70/0GACAQSbhvwd07Ngxtba2Ki8vL9EvBQBIInEP0OLFi9XY2KgvvvhCf//733Xvvfdq6NChuv/+++P9UgCAJBb3H8EdOHBA999/v44cOaJrr71Wt912m7Zt26Zrr7023i8FAEhicQ/QO++8E++nRAo53yn55/LEE094XpOZmel5jST97ne/i2kdYrN161bPa+bMmeN5TVNTk+c1knTdddfFtA4Xh2vBAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmEv4H6YBv+uUvf+l5zdq1az2vKSsr87wGl+a///2v5zWVlZWe1xw+fNjzmlhmk7gYaaJxBAQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATXA0bMfvLX/7ieU1dXV38B+nHjBkzBuR18LWqqirPa2K5sjVSB0dAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJLkaKmL388sue15w8edLzmvLycs9rFi9e7HkNvrZ+/XrPa9asWZOASc42adIkz2vGjx+fgElwqTgCAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMcDFS6I033ohp3SeffBLnSfo3depUz2t8Pl8CJkk+//vf/2Ja9+Mf/zjOk8TP73//e89rrrzyygRMgkvFERAAwAQBAgCY8BygrVu36u6771Z+fr58Pp82bNgQdb9zTs8995zy8vI0fPhwlZaWau/evfGaFwCQIjwHqLu7W0VFRVqxYkW/9y9dulSvvvqqXn/9dW3fvl1XXXWVysrKdOLEiUseFgCQOjyfhFBRUaGKiop+73PO6ZVXXtEzzzyje+65R5L01ltvKTc3Vxs2bNB99913adMCAFJGXD8DamtrU0dHh0pLSyO3BQIBFRcXq6mpqd81PT09CofDURsAIPXFNUAdHR2SpNzc3Kjbc3NzI/edqba2VoFAILIVFBTEcyQAwCBlfhZcTU2NQqFQZNu/f7/1SACAARDXAAWDQUlSZ2dn1O2dnZ2R+87k9/uVkZERtQEAUl9cA1RYWKhgMKj6+vrIbeFwWNu3b1dJSUk8XwoAkOQ8nwV37NgxtbS0RL5ua2vTrl27lJWVpVGjRmnRokV66aWXdMMNN6iwsFDPPvus8vPzNWvWrHjODQBIcp4DtGPHDt1xxx2Rr6urqyVJc+fOVV1dnZ588kl1d3drwYIFOnr0qG677TZt3ryZazEBAKL4nHPOeohvCofDCgQCCoVCfB4Ug97eXs9rpk+fHtNr/fWvf/W85q677vK8ZtOmTZ7XoM/nn38e07rx48fHeZL+net3Cs/nT3/6k+c1I0aM8LwGsbvY7+PmZ8EBAC5PBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMOH5zzFgcHv11Vc9r4nlqtaxmjx5suc1a9eu9bymqanJ8xqp7+9defWjH/3I85qysjLPa3p6ejyvWbx4sec1A+mll17yvIYrW6cOjoAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABNcjDTF7N+/33qE83rxxRetR4i7N954w/OajIwMz2vKy8s9r/nzn//seU2s7rzzTs9rbr755gRMgmTBERAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIKLkaaY9957z3oEXIRwOOx5zdq1axMwSfwUFxd7XjN06NAETIJkwREQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCi5GmmPHjx3te88UXX8R/kHO47rrrPK+pqKjwvObJJ5/0vEaSDh8+7HnNli1bPK958803Pa9pbm72vGYg1dbWel7zj3/8w/Oa++67z/Oa6dOne14jSXl5eTGtw8XhCAgAYIIAAQBMeA7Q1q1bdffddys/P18+n08bNmyIun/evHny+XxRW3l5ebzmBQCkCM8B6u7uVlFRkVasWHHOx5SXl6u9vT2yrVmz5pKGBACkHs8nIVRUVFzwQ2G/369gMBjzUACA1JeQz4AaGhqUk5Ojm266SY8++qiOHDlyzsf29PQoHA5HbQCA1Bf3AJWXl+utt95SfX29fvWrX6mxsVEVFRU6ffp0v4+vra1VIBCIbAUFBfEeCQAwCMX994C+eY7+xIkTNWnSJI0dO1YNDQ2aMWPGWY+vqalRdXV15OtwOEyEAOAykPDTsMeMGaPs7Gy1tLT0e7/f71dGRkbUBgBIfQkP0IEDB3TkyBF+oxgAEMXzj+COHTsWdTTT1tamXbt2KSsrS1lZWXrhhRdUWVmpYDCo1tZWPfnkk7r++utVVlYW18EBAMnNc4B27NihO+64I/L1V5/fzJ07VytXrtTu3bv15ptv6ujRo8rPz9fMmTP1i1/8Qn6/P35TAwCSns8556yH+KZwOKxAIKBQKMTnQTFob2/3vCaWC2NKsV2osbKy0vOaq6++2vOawe7pp5/2vCaWi32iz+LFi2Nat2zZsjhPcnm42O/jXAsOAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJuL+J7lhK5YrVP/85z9PwCSXjyNHjnhes2rVqgRMcrYrr7wypnX19fWe13R0dHhes27dOs9rNm/e7HnNzTff7HkNEo8jIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABBcjBS7RgQMHPK+J5cKdsXjqqadiWveDH/wgzpP0b/bs2QPyOhicOAICAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAExwMVLgEi1btmxAXiczM9Pzmueffz7ucwDxwhEQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCi5EC39DS0uJ5zYYNG+I/SD9KSkoG5HWAgcIREADABAECAJjwFKDa2lrdeuutSk9PV05OjmbNmqXm5uaox5w4cUJVVVW65pprdPXVV6uyslKdnZ1xHRoAkPw8BaixsVFVVVXatm2btmzZolOnTmnmzJnq7u6OPObxxx/X+++/r3Xr1qmxsVEHDx7U7Nmz4z44ACC5eToJYfPmzVFf19XVKScnRzt37tS0adMUCoX0hz/8QatXr9add94pSVq1apXGjx+vbdu26fvf/378JgcAJLVL+gwoFApJkrKysiRJO3fu1KlTp1RaWhp5zLhx4zRq1Cg1NTX1+xw9PT0Kh8NRGwAg9cUcoN7eXi1atEhTp07VhAkTJEkdHR1KS0s762/X5+bmqqOjo9/nqa2tVSAQiGwFBQWxjgQASCIxB6iqqkp79uzRO++8c0kD1NTUKBQKRbb9+/df0vMBAJJDTL+IunDhQm3atElbt27VyJEjI7cHg0GdPHlSR48ejToK6uzsVDAY7Pe5/H6//H5/LGMAAJKYpyMg55wWLlyo9evX68MPP1RhYWHU/ZMnT9awYcNUX18fua25uVn79u3jt7gBAFE8HQFVVVVp9erV2rhxo9LT0yOf6wQCAQ0fPlyBQEAPP/ywqqurlZWVpYyMDD322GMqKSnhDDgAQBRPAVq5cqUkafr06VG3r1q1SvPmzZMk/eY3v9GQIUNUWVmpnp4elZWV6be//W1chgUApA6fc85ZD/FN4XBYgUBAoVBIGRkZ1uPgMvPxxx97XnP77bd7XnPmmaIX49NPP/W85swfkwMD4WK/j3MtOACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJiI6S+iAqnq7bffHpDXueuuuzyv4crWSDUcAQEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJrgYKVJSW1tbTOvq6uo8r8nOzva85te//rXnNUCq4QgIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDBxUiRkpqbm2Nad+LECc9rKioqPK8JBoOe1wCphiMgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEFyNFSsrLy4tpXVpamuc1ixcvjum1gMsdR0AAABMECABgwlOAamtrdeuttyo9PV05OTmaNWvWWX93Zfr06fL5fFHbI488EtehAQDJz1OAGhsbVVVVpW3btmnLli06deqUZs6cqe7u7qjHzZ8/X+3t7ZFt6dKlcR0aAJD8PJ2EsHnz5qiv6+rqlJOTo507d2ratGmR20eMGMFffAQAnNclfQYUCoUkSVlZWVG3v/3228rOztaECRNUU1Oj48ePn/M5enp6FA6HozYAQOqL+TTs3t5eLVq0SFOnTtWECRMitz/wwAMaPXq08vPztXv3bj311FNqbm7We++91+/z1NbW6oUXXoh1DABAkoo5QFVVVdqzZ48+/vjjqNsXLFgQ+ffEiROVl5enGTNmqLW1VWPHjj3reWpqalRdXR35OhwOq6CgINaxAABJIqYALVy4UJs2bdLWrVs1cuTI8z62uLhYktTS0tJvgPx+v/x+fyxjAACSmKcAOef02GOPaf369WpoaFBhYeEF1+zatUtS7L+ZDgBITZ4CVFVVpdWrV2vjxo1KT09XR0eHJCkQCGj48OFqbW3V6tWr9cMf/lDXXHONdu/erccff1zTpk3TpEmTEvIfAABITp4CtHLlSkl9v2z6TatWrdK8efOUlpamDz74QK+88oq6u7tVUFCgyspKPfPMM3EbGACQGjz/CO58CgoK1NjYeEkDAQAuD1wNGympqKgopnU9PT1xngTAuXAxUgCACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAExcYT3AmZxzkqRwOGw8CQAgFl99//7q+/m5DLoAdXV1SZIKCgqMJwEAXIquri4FAoFz3u9zF0rUAOvt7dXBgweVnp4un88XdV84HFZBQYH279+vjIwMowntsR/6sB/6sB/6sB/6DIb94JxTV1eX8vPzNWTIuT/pGXRHQEOGDNHIkSPP+5iMjIzL+g32FfZDH/ZDH/ZDH/ZDH+v9cL4jn69wEgIAwAQBAgCYSKoA+f1+LVmyRH6/33oUU+yHPuyHPuyHPuyHPsm0HwbdSQgAgMtDUh0BAQBSBwECAJggQAAAEwQIAGAiaQK0YsUKXXfddbryyitVXFysTz75xHqkAff888/L5/NFbePGjbMeK+G2bt2qu+++W/n5+fL5fNqwYUPU/c45Pffcc8rLy9Pw4cNVWlqqvXv32gybQBfaD/PmzTvr/VFeXm4zbILU1tbq1ltvVXp6unJycjRr1iw1NzdHPebEiROqqqrSNddco6uvvlqVlZXq7Ow0mjgxLmY/TJ8+/az3wyOPPGI0cf+SIkDvvvuuqqurtWTJEn366acqKipSWVmZDh06ZD3agLvlllvU3t4e2T7++GPrkRKuu7tbRUVFWrFiRb/3L126VK+++qpef/11bd++XVdddZXKysp04sSJAZ40sS60HySpvLw86v2xZs2aAZww8RobG1VVVaVt27Zpy5YtOnXqlGbOnKnu7u7IYx5//HG9//77WrdunRobG3Xw4EHNnj3bcOr4u5j9IEnz58+Pej8sXbrUaOJzcElgypQprqqqKvL16dOnXX5+vqutrTWcauAtWbLEFRUVWY9hSpJbv3595Ove3l4XDAbdsmXLIrcdPXrU+f1+t2bNGoMJB8aZ+8E55+bOnevuuecek3msHDp0yElyjY2Nzrm+/+2HDRvm1q1bF3nMv/71LyfJNTU1WY2ZcGfuB+ec+7//+z/305/+1G6oizDoj4BOnjypnTt3qrS0NHLbkCFDVFpaqqamJsPJbOzdu1f5+fkaM2aMHnzwQe3bt896JFNtbW3q6OiIen8EAgEVFxdflu+PhoYG5eTk6KabbtKjjz6qI0eOWI+UUKFQSJKUlZUlSdq5c6dOnToV9X4YN26cRo0aldLvhzP3w1fefvttZWdna8KECaqpqdHx48ctxjunQXcx0jMdPnxYp0+fVm5ubtTtubm5+vzzz42mslFcXKy6ujrddNNNam9v1wsvvKDbb79de/bsUXp6uvV4Jjo6OiSp3/fHV/ddLsrLyzV79mwVFhaqtbVVTz/9tCoqKtTU1KShQ4dajxd3vb29WrRokaZOnaoJEyZI6ns/pKWlKTMzM+qxqfx+6G8/SNIDDzyg0aNHKz8/X7t379ZTTz2l5uZmvffee4bTRhv0AcLXKioqIv+eNGmSiouLNXr0aK1du1YPP/yw4WQYDO67777IvydOnKhJkyZp7Nixamho0IwZMwwnS4yqqirt2bPnsvgc9HzOtR8WLFgQ+ffEiROVl5enGTNmqLW1VWPHjh3oMfs16H8El52draFDh551FktnZ6eCwaDRVINDZmambrzxRrW0tFiPYuar9wDvj7ONGTNG2dnZKfn+WLhwoTZt2qSPPvoo6s+3BINBnTx5UkePHo16fKq+H861H/pTXFwsSYPq/TDoA5SWlqbJkyervr4+cltvb6/q6+tVUlJiOJm9Y8eOqbW1VXl5edajmCksLFQwGIx6f4TDYW3fvv2yf38cOHBAR44cSan3h3NOCxcu1Pr16/Xhhx+qsLAw6v7Jkydr2LBhUe+H5uZm7du3L6XeDxfaD/3ZtWuXJA2u94P1WRAX45133nF+v9/V1dW5f/7zn27BggUuMzPTdXR0WI82oH72s5+5hoYG19bW5v72t7+50tJSl52d7Q4dOmQ9WkJ1dXW5zz77zH322WdOklu+fLn77LPP3L///W/nnHMvv/yyy8zMdBs3bnS7d+9299xzjyssLHRffvml8eTxdb790NXV5RYvXuyamppcW1ub++CDD9x3v/tdd8MNN7gTJ05Yjx43jz76qAsEAq6hocG1t7dHtuPHj0ce88gjj7hRo0a5Dz/80O3YscOVlJS4kpISw6nj70L7oaWlxb344otux44drq2tzW3cuNGNGTPGTZs2zXjyaEkRIOece+2119yoUaNcWlqamzJlitu2bZv1SANuzpw5Li8vz6Wlpblvf/vbbs6cOa6lpcV6rIT76KOPnKSztrlz5zrn+k7FfvbZZ11ubq7z+/1uxowZrrm52XboBDjffjh+/LibOXOmu/baa92wYcPc6NGj3fz581Pu/6T1998vya1atSrymC+//NL95Cc/cd/61rfciBEj3L333uva29vthk6AC+2Hffv2uWnTprmsrCzn9/vd9ddf75544gkXCoVsBz8Df44BAGBi0H8GBABITQQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACAif8HWQykys9wAxgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## \n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "print(images.shape, labels.shape)\n",
    "\n",
    "## show an image\n",
    "plt.imshow(images[0].numpy().squeeze(), cmap='gray_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6889d575",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## display more images\n",
    "# figure = plt.figure()\n",
    "# num_of_images = 60\n",
    "# for index in range(1, num_of_images + 1):\n",
    "#     plt.subplot(6, 10, index)\n",
    "#     plt.axis('off')\n",
    "#     plt.imshow(images[index].numpy().squeeze(), cmap='gray_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fcbd17c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ReLU\n",
    "def relu(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "# Derivative of the relu \n",
    "def der_relu(x):\n",
    "    return 1.0*(x>0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bab1c0d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object Module.parameters at 0x7f2f93eae120>\n"
     ]
    }
   ],
   "source": [
    "## build the neural network with two hidden layers\n",
    "input_size = 784\n",
    "hidden_sizes = [128, 64]\n",
    "output_size = 10\n",
    "\n",
    "model = nn.Sequential(nn.Linear(input_size, hidden_sizes[0]),\n",
    "                      nn.ReLU(),\n",
    "                      nn.Linear(hidden_sizes[0], hidden_sizes[1]),\n",
    "                      nn.ReLU(),\n",
    "                      nn.Linear(hidden_sizes[1], output_size),\n",
    "                      nn.LogSoftmax(dim=1))\n",
    "print(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4e2fad5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 784])\n",
      "torch.Size([128])\n",
      "torch.Size([64, 128])\n",
      "torch.Size([64])\n",
      "torch.Size([10, 64])\n",
      "torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "for layer in model.parameters():\n",
    "    print(layer.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7a36800b",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss()\n",
    "images, labels = next(iter(trainloader))\n",
    "images = images.view(images.shape[0], -1)\n",
    "\n",
    "logps = model(images) #log probabilities\n",
    "loss = criterion(logps, labels) #calculate the NLL loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d71e6540",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 - Training loss: 2.250600530902968, Valid loss: 2.1845455169677734, valid accuracy: 0.3172\n",
      "Epoch 5 - Training loss: 0.8707033729375299, Valid loss: 0.7667863965034485, valid accuracy: 0.8127\n",
      "Epoch 10 - Training loss: 0.4946035433933933, Valid loss: 0.4630560278892517, valid accuracy: 0.8745\n",
      "Epoch 15 - Training loss: 0.4016501232981682, Valid loss: 0.38223323225975037, valid accuracy: 0.8928\n",
      "Epoch 20 - Training loss: 0.35999391590163654, Valid loss: 0.34338465332984924, valid accuracy: 0.9026\n",
      "Epoch 25 - Training loss: 0.3347539530316396, Valid loss: 0.3221394121646881, valid accuracy: 0.9063\n",
      "Epoch 30 - Training loss: 0.3171451529190103, Valid loss: 0.30577144026756287, valid accuracy: 0.9112\n",
      "Epoch 35 - Training loss: 0.3030662423909219, Valid loss: 0.2920578718185425, valid accuracy: 0.9167\n",
      "Epoch 40 - Training loss: 0.2914284908933553, Valid loss: 0.2823936939239502, valid accuracy: 0.9196\n",
      "Epoch 45 - Training loss: 0.2808937486539136, Valid loss: 0.2745712697505951, valid accuracy: 0.9209\n",
      "Epoch 50 - Training loss: 0.2715132375801804, Valid loss: 0.26508647203445435, valid accuracy: 0.9251\n",
      "Epoch 55 - Training loss: 0.26214790320806286, Valid loss: 0.25605708360671997, valid accuracy: 0.9269\n",
      "Epoch 60 - Training loss: 0.2534391669703445, Valid loss: 0.24969513714313507, valid accuracy: 0.928\n",
      "Epoch 65 - Training loss: 0.2450970552766374, Valid loss: 0.24235180020332336, valid accuracy: 0.9311\n",
      "Epoch 70 - Training loss: 0.2368967719495233, Valid loss: 0.2336970716714859, valid accuracy: 0.9316\n",
      "Epoch 75 - Training loss: 0.2291177587627348, Valid loss: 0.22755631804466248, valid accuracy: 0.9352\n",
      "Epoch 80 - Training loss: 0.22163150009951357, Valid loss: 0.22288267314434052, valid accuracy: 0.9352\n",
      "Epoch 85 - Training loss: 0.2143436432289861, Valid loss: 0.2148040235042572, valid accuracy: 0.9366\n",
      "Epoch 90 - Training loss: 0.20739914594031472, Valid loss: 0.20834773778915405, valid accuracy: 0.9394\n",
      "Epoch 95 - Training loss: 0.20063189439761486, Valid loss: 0.20536789298057556, valid accuracy: 0.9394\n",
      "Epoch 100 - Training loss: 0.1942732712305558, Valid loss: 0.19615505635738373, valid accuracy: 0.9422\n",
      "Epoch 105 - Training loss: 0.1881702178433887, Valid loss: 0.1906653493642807, valid accuracy: 0.9423\n",
      "Epoch 110 - Training loss: 0.18249149006932402, Valid loss: 0.18513476848602295, valid accuracy: 0.9436\n",
      "Epoch 115 - Training loss: 0.17694933231927948, Valid loss: 0.18136066198349, valid accuracy: 0.9466\n",
      "Epoch 120 - Training loss: 0.17176754436473538, Valid loss: 0.1766618937253952, valid accuracy: 0.9477\n",
      "Epoch 125 - Training loss: 0.1668320977524209, Valid loss: 0.17297282814979553, valid accuracy: 0.9488\n",
      "Epoch 130 - Training loss: 0.16211501371735004, Valid loss: 0.1680474728345871, valid accuracy: 0.9494\n",
      "Epoch 135 - Training loss: 0.15757711620123654, Valid loss: 0.164456307888031, valid accuracy: 0.9513\n",
      "Epoch 140 - Training loss: 0.15338309211477733, Valid loss: 0.1615661382675171, valid accuracy: 0.9514\n",
      "Epoch 145 - Training loss: 0.14932831922875667, Valid loss: 0.15782862901687622, valid accuracy: 0.9529\n",
      "Epoch 150 - Training loss: 0.1452864858526379, Valid loss: 0.15302783250808716, valid accuracy: 0.9546\n",
      "Epoch 155 - Training loss: 0.14166113656800566, Valid loss: 0.149844691157341, valid accuracy: 0.955\n",
      "Epoch 160 - Training loss: 0.13801426839814193, Valid loss: 0.1476271152496338, valid accuracy: 0.9563\n",
      "Epoch 165 - Training loss: 0.13460647090594333, Valid loss: 0.14550794661045074, valid accuracy: 0.9569\n",
      "Epoch 170 - Training loss: 0.13136780830715766, Valid loss: 0.14250710606575012, valid accuracy: 0.9582\n",
      "Epoch 175 - Training loss: 0.12813783046017005, Valid loss: 0.13910123705863953, valid accuracy: 0.9587\n",
      "Epoch 180 - Training loss: 0.12501218658822305, Valid loss: 0.1367349773645401, valid accuracy: 0.9592\n",
      "Epoch 185 - Training loss: 0.12234286631324462, Valid loss: 0.13460810482501984, valid accuracy: 0.9594\n",
      "Epoch 190 - Training loss: 0.1195290442973153, Valid loss: 0.1330816000699997, valid accuracy: 0.9601\n",
      "Epoch 195 - Training loss: 0.11669512347443335, Valid loss: 0.13188868761062622, valid accuracy: 0.9622\n",
      "Epoch 200 - Training loss: 0.11410011862045222, Valid loss: 0.1283601075410843, valid accuracy: 0.9618\n",
      "Epoch 205 - Training loss: 0.11155691305930013, Valid loss: 0.1263616979122162, valid accuracy: 0.9629\n",
      "Epoch 210 - Training loss: 0.10922265082184694, Valid loss: 0.12465989589691162, valid accuracy: 0.9627\n",
      "Epoch 215 - Training loss: 0.10689532955381661, Valid loss: 0.12265043705701828, valid accuracy: 0.9642\n",
      "Epoch 220 - Training loss: 0.10459686298646144, Valid loss: 0.12240558117628098, valid accuracy: 0.9637\n",
      "Epoch 225 - Training loss: 0.10240499235427519, Valid loss: 0.12033969163894653, valid accuracy: 0.9644\n",
      "Epoch 230 - Training loss: 0.100190045052905, Valid loss: 0.11782924085855484, valid accuracy: 0.9652\n",
      "Epoch 235 - Training loss: 0.09829165601233111, Valid loss: 0.1170019879937172, valid accuracy: 0.9658\n",
      "Epoch 240 - Training loss: 0.09616526105165132, Valid loss: 0.11520896106958389, valid accuracy: 0.9649\n",
      "Epoch 245 - Training loss: 0.09435829380129986, Valid loss: 0.11404921859502792, valid accuracy: 0.966\n",
      "Epoch 250 - Training loss: 0.09245389091994749, Valid loss: 0.11527825891971588, valid accuracy: 0.9666\n",
      "Epoch 255 - Training loss: 0.09074870619311262, Valid loss: 0.11243213713169098, valid accuracy: 0.9663\n",
      "Epoch 260 - Training loss: 0.08886774303391576, Valid loss: 0.1112363189458847, valid accuracy: 0.9663\n",
      "Epoch 265 - Training loss: 0.08718817191980859, Valid loss: 0.11069852858781815, valid accuracy: 0.9667\n",
      "Epoch 270 - Training loss: 0.0856043607232842, Valid loss: 0.10861452668905258, valid accuracy: 0.9672\n",
      "Epoch 275 - Training loss: 0.0840213514620196, Valid loss: 0.10687361657619476, valid accuracy: 0.9673\n",
      "Epoch 280 - Training loss: 0.08248704167078934, Valid loss: 0.10637955367565155, valid accuracy: 0.9677\n",
      "Epoch 285 - Training loss: 0.08090602816132181, Valid loss: 0.10526321828365326, valid accuracy: 0.9678\n",
      "Epoch 290 - Training loss: 0.0793841340457961, Valid loss: 0.10441949963569641, valid accuracy: 0.9677\n",
      "Epoch 295 - Training loss: 0.07790897675811737, Valid loss: 0.10310375690460205, valid accuracy: 0.9688\n",
      "Epoch 300 - Training loss: 0.07650965481384922, Valid loss: 0.10184430330991745, valid accuracy: 0.9683\n",
      "Epoch 305 - Training loss: 0.07517121951860278, Valid loss: 0.10139989107847214, valid accuracy: 0.9689\n",
      "Epoch 310 - Training loss: 0.07385896057980672, Valid loss: 0.10046019405126572, valid accuracy: 0.9689\n",
      "Epoch 315 - Training loss: 0.07250533159822226, Valid loss: 0.09986236691474915, valid accuracy: 0.969\n",
      "Epoch 320 - Training loss: 0.07126498964132626, Valid loss: 0.09876611083745956, valid accuracy: 0.9695\n",
      "Epoch 325 - Training loss: 0.06999648785365543, Valid loss: 0.09807785600423813, valid accuracy: 0.9689\n",
      "Epoch 330 - Training loss: 0.06882353277722084, Valid loss: 0.09759266674518585, valid accuracy: 0.9696\n",
      "Epoch 335 - Training loss: 0.06766018345366234, Valid loss: 0.09733372926712036, valid accuracy: 0.9691\n",
      "Epoch 340 - Training loss: 0.06639244003412026, Valid loss: 0.09907132387161255, valid accuracy: 0.9704\n",
      "Epoch 345 - Training loss: 0.06531694428330022, Valid loss: 0.09608533978462219, valid accuracy: 0.9709\n",
      "Epoch 350 - Training loss: 0.06419373887912758, Valid loss: 0.09431175887584686, valid accuracy: 0.9698\n",
      "Epoch 355 - Training loss: 0.063099695300596, Valid loss: 0.0943971499800682, valid accuracy: 0.971\n",
      "Epoch 360 - Training loss: 0.062100667998925455, Valid loss: 0.09370095282793045, valid accuracy: 0.9714\n",
      "Epoch 365 - Training loss: 0.061056437869387456, Valid loss: 0.09264546632766724, valid accuracy: 0.971\n",
      "Epoch 370 - Training loss: 0.060129907794856725, Valid loss: 0.09228863567113876, valid accuracy: 0.971\n",
      "Epoch 375 - Training loss: 0.059190179427354925, Valid loss: 0.09122461825609207, valid accuracy: 0.9708\n",
      "Epoch 380 - Training loss: 0.05819955792080468, Valid loss: 0.09067273885011673, valid accuracy: 0.9709\n",
      "Epoch 385 - Training loss: 0.057224580434772535, Valid loss: 0.09116344898939133, valid accuracy: 0.972\n",
      "Epoch 390 - Training loss: 0.05636768751958413, Valid loss: 0.09029640257358551, valid accuracy: 0.972\n",
      "Epoch 395 - Training loss: 0.05543757891649805, Valid loss: 0.08966539800167084, valid accuracy: 0.9715\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 400 - Training loss: 0.05454065876681286, Valid loss: 0.08916892856359482, valid accuracy: 0.9719\n",
      "Epoch 405 - Training loss: 0.0537037297066595, Valid loss: 0.08984105288982391, valid accuracy: 0.9727\n",
      "Epoch 410 - Training loss: 0.05281362741458804, Valid loss: 0.08865058422088623, valid accuracy: 0.9723\n",
      "Epoch 415 - Training loss: 0.05214938601758927, Valid loss: 0.08876809477806091, valid accuracy: 0.9728\n",
      "Epoch 420 - Training loss: 0.0511502928680369, Valid loss: 0.08788720518350601, valid accuracy: 0.9725\n",
      "Epoch 425 - Training loss: 0.05043139192424635, Valid loss: 0.08795351535081863, valid accuracy: 0.9728\n",
      "Epoch 430 - Training loss: 0.049638301911039046, Valid loss: 0.08869679272174835, valid accuracy: 0.9723\n",
      "Epoch 435 - Training loss: 0.048917424625782635, Valid loss: 0.0867656022310257, valid accuracy: 0.9727\n",
      "Epoch 440 - Training loss: 0.048205944730826754, Valid loss: 0.08724566549062729, valid accuracy: 0.9727\n",
      "Epoch 445 - Training loss: 0.04745398019688692, Valid loss: 0.08708969503641129, valid accuracy: 0.9727\n",
      "Epoch 450 - Training loss: 0.04671811185993636, Valid loss: 0.0855935737490654, valid accuracy: 0.9736\n",
      "Epoch 455 - Training loss: 0.04606161472836394, Valid loss: 0.08552189916372299, valid accuracy: 0.9729\n",
      "Epoch 460 - Training loss: 0.045364310519138314, Valid loss: 0.08531999588012695, valid accuracy: 0.9726\n",
      "Epoch 465 - Training loss: 0.04474312287848641, Valid loss: 0.08554147928953171, valid accuracy: 0.9735\n",
      "Epoch 470 - Training loss: 0.04400672540113306, Valid loss: 0.08459219336509705, valid accuracy: 0.9726\n",
      "Epoch 475 - Training loss: 0.043360543843390525, Valid loss: 0.08419235050678253, valid accuracy: 0.9728\n",
      "Epoch 480 - Training loss: 0.04274156628730598, Valid loss: 0.08468344062566757, valid accuracy: 0.9732\n",
      "Epoch 485 - Training loss: 0.04205921460250849, Valid loss: 0.084197498857975, valid accuracy: 0.9733\n",
      "Epoch 490 - Training loss: 0.041411171585365114, Valid loss: 0.08464943617582321, valid accuracy: 0.9738\n",
      "Epoch 495 - Training loss: 0.0408226371383362, Valid loss: 0.08329398185014725, valid accuracy: 0.9739\n",
      "Training Time (in minutes) = 140.7386125524839\n"
     ]
    }
   ],
   "source": [
    "## training process\n",
    "#optimizer = optim.SGD(model.parameters(), lr=0.003, momentum=0.9)\n",
    "time0 = time()\n",
    "lr = 0.0008\n",
    "\n",
    "train_list = []\n",
    "valid_list = []\n",
    "\n",
    "min_loss = np.inf\n",
    "epochs = 500\n",
    "for e in range(epochs):\n",
    "    running_loss = 0\n",
    "    for images, labels in trainloader:\n",
    "        # Flatten MNIST images into a 784 long vector\n",
    "        images = images.view(images.shape[0], -1)\n",
    "    \n",
    "        # Training pass\n",
    "        #optimizer.zero_grad()\n",
    "        \n",
    "        output = model(images)\n",
    "        loss = criterion(output, labels)\n",
    "        \n",
    "        #This is where the model learns by backpropagating\n",
    "        loss.backward()\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            for param in model.parameters():\n",
    "                param -= lr*param.grad\n",
    "\n",
    "        #And optimizes its weights here\n",
    "        #optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()\n",
    "    \n",
    "        model.zero_grad()\n",
    "    #else:\n",
    "    #    print(\"Epoch {} - Training loss: {}\".format(e, running_loss/len(trainloader)))\n",
    "        \n",
    "    ## valid each epoch\n",
    "    correct_count, all_count = 0, 0\n",
    "    valid_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for images,labels in valloader:\n",
    "            img = images.view(images.shape[0], -1)\n",
    "            output = model(img)\n",
    "            # print(output.shape)\n",
    "            loss = criterion(output, labels)\n",
    "            valid_loss += loss\n",
    "            for i in range(len(labels)):\n",
    "                img = images[i].view(1, 784)\n",
    "                with torch.no_grad():\n",
    "                    logps = model(img)\n",
    "                ps = torch.exp(logps)\n",
    "                probab = list(ps.numpy()[0])\n",
    "                pred_label = probab.index(max(probab))\n",
    "                true_label = labels.numpy()[i]\n",
    "                if(true_label == pred_label):\n",
    "                    correct_count += 1\n",
    "                all_count += 1\n",
    "    \n",
    "    \n",
    "    tmp_train_loss = running_loss/len(trainloader)\n",
    "    tmp_valid_loss = valid_loss/len(valloader)\n",
    "    \n",
    "    train_list.append(tmp_train_loss)\n",
    "    valid_list.append(tmp_valid_loss)\n",
    "    \n",
    "    if min_loss > tmp_valid_loss:\n",
    "        bst_model = model\n",
    "    if e%5 == 0: \n",
    "        print(\"Epoch {} - Training loss: {}, Valid loss: {}, valid accuracy: {}\".format(e, tmp_train_loss, tmp_valid_loss, correct_count/all_count))\n",
    "    # print(\"Number Of Images Tested =\", all_count)\n",
    "    #print(\"Valid Model Accuracy =\", (correct_count/all_count))\n",
    "    \n",
    "    \n",
    "print(\"Training Time (in minutes) =\",(time()-time0)/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8b400ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save best model\n",
    "torch.save(bst_model, './my_mnist_model.pt') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cdfbd33c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABfSUlEQVR4nO3deXhU5eH28e+ZSTLZF0hIAiQEBLIgi4BgUBELirhUkCqlqIC7okjVWql1b0VrbanVV6tWqFrE5SdIFYUIuKCIrIqQRFAgQZJAgCxkT+a8fwwZE0hCEmbJcn96zQVzzjNnnjmJnZtnNUzTNBERERHpICzeroCIiIiIKynciIiISIeicCMiIiIdisKNiIiIdCgKNyIiItKhKNyIiIhIh6JwIyIiIh2Kwo2IiIh0KAo3IiIi0qEo3Ih0MDNmzCAhIcHb1WjUwoULMQyDPXv2OI+NGTOGMWPGnPS1n3zyCYZh8Mknn7itfiLS/inciHiIYRjNerSVL+6qqioiIyM555xzGi1jmiZxcXEMHTrUgzU7udoAtXHjRm9XpVm2bt3K1VdfTVxcHDabjS5dujBu3DgWLFhATU2Nt6sn0u74eLsCIp3Fa6+9Vu/5q6++Slpa2gnHk5OTT+l9XnrpJex2+yldA8DX15crr7ySf/3rX+zdu5devXqdUOazzz5j3759/Pa3vz2l91q5cuUpvb49e/nll7nllluIjo7mmmuuoV+/fhQXF7Nq1Squv/56cnJy+MMf/uDtaoq0Kwo3Ih5y9dVX13v+1VdfkZaWdsLx45WWlhIYGNjs9/H19W1V/Roybdo0XnjhBd544w3uu+++E84vWrQIi8XCr3/961N6Hz8/v1N6fXv11Vdfccstt5Camsry5csJCQlxnpszZw4bN27ku+++c8l7lZSUEBQU5JJribR16pYSaUPGjBnD6aefzqZNmxg9ejSBgYHOf7W/9957XHLJJXTv3h2bzcZpp53GY489dkK3xfFjbvbs2YNhGPz1r3/lxRdf5LTTTsNms3HmmWeyYcOGJutz9tlnk5CQwKJFi044V1VVxTvvvMP5559P9+7d+fbbb5kxYwZ9+vTB39+fmJgYrrvuOg4dOtSsz338mJt9+/YxceJEgoKC6NatG7/97W+pqKg46bVaYsuWLUyYMIHQ0FCCg4MZO3YsX331Vb0yVVVVPPLII/Tr1w9/f3+6du3KOeecQ1pamrNMbm4uM2fOpGfPnthsNmJjY7n88svrjStqyCOPPIJhGPz3v/+tF2xqDR8+nBkzZgCNjzeq/fkuXLjQeWzGjBkEBwfzww8/cPHFFxMSEsK0adO4/fbbCQ4OprS09IT3mjp1KjExMfV+nz788EPOPfdcgoKCCAkJ4ZJLLmH79u1NfiaRtkAtNyJtzKFDh5gwYQK//vWvufrqq4mOjgYc40iCg4O56667CA4OZvXq1Tz44IMUFRXx1FNPnfS6ixYtori4mJtvvhnDMPjLX/7CFVdcwY8//thoa49hGPzmN7/h8ccfZ/v27QwYMMB57qOPPuLw4cNMmzYNgLS0NH788UdmzpxJTEwM27dv58UXX2T79u189dVXGIbR7HtQVlbG2LFjycrKYvbs2XTv3p3XXnuN1atXN/saJ7N9+3bOPfdcQkNDuffee/H19eVf//oXY8aM4dNPP2XkyJEAPPzww8ybN48bbriBESNGUFRUxMaNG9m8eTMXXHABAJMnT2b79u3ccccdJCQkcODAAdLS0sjKymp0cHdpaSmrVq1i9OjRxMfHu+xz1aqurmb8+PGcc845/PWvfyUwMJCEhASee+45PvjgA6688sp6dfnf//7HjBkzsFqtgKMbdfr06YwfP54nn3yS0tJSnn/+ec455xy2bNnSpgeti2CKiFfMmjXLPP4/wfPOO88EzBdeeOGE8qWlpSccu/nmm83AwECzvLzceWz69Olmr169nM93795tAmbXrl3Nw4cPO4+/9957JmD+73//a7Ke27dvNwFz7ty59Y7/+te/Nv39/c3CwsJG6/fGG2+YgPnZZ585jy1YsMAEzN27d9f73Oedd57z+fz5803AfOutt5zHSkpKzL59+5qAuWbNmibrXPseGzZsaLTMxIkTTT8/P/OHH35wHtu/f78ZEhJijh492nls8ODB5iWXXNLodY4cOWIC5lNPPdVknY73zTffmIB55513Nqv8mjVrGvzstT/fBQsWOI9Nnz7dBMz77ruvXlm73W726NHDnDx5cr3jb731Vr2fU3FxsRkeHm7eeOON9crl5uaaYWFhJxwXaWvULSXSxthsNmbOnHnC8YCAAOffi4uLyc/P59xzz6W0tJSMjIyTXnfKlClEREQ4n5977rkA/Pjjj02+LiUlhTPOOIPFixc7j5WUlLBs2TIuvfRSQkNDT6hfeXk5+fn5nHXWWQBs3rz5pPWra/ny5cTGxvKrX/3KeSwwMJCbbrqpRddpTE1NDStXrmTixIn06dPHeTw2Npbf/OY3rF27lqKiIgDCw8PZvn07O3fubPBaAQEB+Pn58cknn3DkyJFm16H2+g11R7nKrbfeWu+5YRhceeWVLF++nKNHjzqPv/nmm/To0cM5My4tLY2CggKmTp1Kfn6+82G1Whk5ciRr1qxxW51FXEHhRqSN6dGjR4MDbLdv386kSZMICwsjNDSUqKgo52DkwsLCk173+K6P2qBT+4VcVlZGbm5uvUetadOmsXv3br788ksAli5dSmlpqbNLCuDw4cPceeedREdHExAQQFRUFL179252/erau3cvffv2PaErKzExsUXXaczBgwcpLS1t8HrJycnY7Xays7MBePTRRykoKKB///4MHDiQ3/3ud3z77bfO8jabjSeffJIPP/yQ6OhoRo8ezV/+8pd6968htaGwuLjYJZ/peD4+PvTs2fOE41OmTKGsrIxly5YBcPToUZYvX86VV17pvN+1Qe4Xv/gFUVFR9R4rV67kwIEDbqmziKso3Ii0MXVbQGoVFBRw3nnn8c033/Doo4/yv//9j7S0NJ588kmAZk39rh1LcTzTNAHHv95jY2PrPWpNnToVi8XiHFi8aNEiIiIiuPjii51lrrrqKl566SVuueUW3n33XVauXMlHH33U7Pq1VaNHj+aHH37glVde4fTTT+fll19m6NChvPzyy84yc+bM4fvvv2fevHn4+/vzwAMPkJyczJYtWxq9bt++ffHx8WHbtm3NqkdjY5YaWwfHZrNhsZz4f/FnnXUWCQkJvPXWWwD873//o6ysjClTpjjL1P68XnvtNdLS0k54vPfee82qs4i3aECxSDvwySefcOjQId59911Gjx7tPL57926Xvcf48ePrzQCqq3v37px//vm8/fbbPPDAA6SlpTFjxgxnC9ORI0dYtWoVjzzyCA8++KDzdY115ZxMr169+O677zBNs96XemZmZquud7yoqCgCAwMbvF5GRgYWi4W4uDjnsS5dujBz5kxmzpzJ0aNHGT16NA8//DA33HCDs8xpp53G3Xffzd13383OnTsZMmQITz/9NK+//nqDdQgMDOQXv/gFq1evJjs7u977NaS2pa2goKDe8b179zb3YztdddVV/OMf/6CoqIg333yThIQEZxdi7WcB6NatG+PGjWvx9UW8TS03Iu1AbatLbSsLQGVlJf/v//0/l71HbGws48aNq/eoa9q0aRw4cICbb76Zqqqqel1SDdUPYP78+a2qy8UXX8z+/ft55513nMdKS0t58cUXW3W941mtVi688ELee++9etO18/LyWLRoEeecc46z2+j4qezBwcH07dvXOS29tLSU8vLyemVOO+00QkJCTjp1/aGHHsI0Ta655pp6Y2Bqbdq0if/85z+AI/BZrVY+++yzemVa8zswZcoUKioq+M9//sNHH33EVVddVe/8+PHjCQ0N5fHHH6eqquqE1x88eLDF7yniSWq5EWkHRo0aRUREBNOnT2f27NkYhsFrr712Qphwp8mTJ3Pbbbfx3nvvERcXV68FKTQ01DnWpKqqih49erBy5cpWtyzdeOONPPvss1x77bVs2rSJ2NhYXnvttRYtZgjwyiuvOLvG6rrzzjv505/+RFpaGueccw633XYbPj4+/Otf/6KiooK//OUvzrIpKSmMGTOGYcOG0aVLFzZu3Mg777zD7bffDsD333/P2LFjueqqq0hJScHHx4clS5aQl5d30sUNR40axXPPPcdtt91GUlJSvRWKP/nkE5YtW8af/vQnAMLCwrjyyiv55z//iWEYnHbaabz//vutGv8ydOhQ+vbty/33309FRUW9Lilw/Dyff/55rrnmGoYOHcqvf/1roqKiyMrK4oMPPuDss8/m2WefbfH7iniMN6dqiXRmjU0FHzBgQIPlv/jiC/Oss84yAwICzO7du5v33nuvuWLFihOmBzc2FbyhqcqA+dBDDzW7zldeeaUJmPfee+8J5/bt22dOmjTJDA8PN8PCwswrr7zS3L9//wnv0Zyp4KZpmnv37jV/+ctfmoGBgWZkZKR55513mh999FGLpoI39sjOzjZN0zQ3b95sjh8/3gwODjYDAwPN888/3/zyyy/rXetPf/qTOWLECDM8PNwMCAgwk5KSzD//+c9mZWWlaZqmmZ+fb86aNctMSkoyg4KCzLCwMHPkyJH1prGfzKZNm8zf/OY3Zvfu3U1fX18zIiLCHDt2rPmf//zHrKmpcZY7ePCgOXnyZDMwMNCMiIgwb775ZvO7775rcCp4UFBQk+95//33m4DZt2/fRsusWbPGHD9+vBkWFmb6+/ubp512mjljxgxz48aNzf5sIt5gmKYH/+knIiIi4mYacyMiIiIdisKNiIiIdCgKNyIiItKhKNyIiIhIh6JwIyIiIh2Kwo2IiIh0KJ1uET+73c7+/fsJCQlpdK8WERERaVtM06S4uJju3bs3uG9aXZ0u3Ozfv/+ke7iIiIhI25Sdnd3gjvd1dbpwExISAjhuTu3eMSIiItK2FRUVERcX5/web0qnCze1XVGhoaEKNyIiIu1Mc4aUaECxiIiIdCgKNyIiItKhKNyIiIhIh9LpxtyIiEjHZbfbqays9HY1pJX8/PxOOs27ORRuRESkQ6isrGT37t3Y7XZvV0VayWKx0Lt3b/z8/E7pOgo3IiLS7pmmSU5ODlarlbi4OJf86188q3aR3ZycHOLj409poV2FGxERafeqq6spLS2le/fuBAYGers60kpRUVHs37+f6upqfH19W30dRVsREWn3ampqAE65O0O8q/bnV/vzbC2FGxER6TC0Z2D75qqfn7qlTlVWFuTnN34+MhLi4z1XHxERkU5O4eZUZGVBYiKUlzdext8fMjMVcERExCMSEhKYM2cOc+bM8eo1vEnh5lTk5zcdbMBxPj9f4UZEpC3zQiv8ybpgHnroIR5++OEWX3fDhg0EBQW1slYdg8KNiIh0bl5qhc/JyXH+/c033+TBBx8kMzPTeSw4ONj5d9M0qampwcfn5F/bUVFRLqtje6UBxSIi0rm1pBXehWJiYpyPsLAwDMNwPs/IyCAkJIQPP/yQYcOGYbPZWLt2LT/88AOXX3450dHRBAcHc+aZZ/Lxxx/Xu25CQgLz5893PjcMg5dffplJkyYRGBhIv379WLZsWYvqmpWVxeWXX05wcDChoaFcddVV5OXlOc9/8803nH/++YSEhBAaGsqwYcPYuHEjAHv37uWyyy4jIiKCoKAgBgwYwPLly1t/45pB4UZERDoe04SSkuY9ysqad82ysuZdzzRd9jHuu+8+nnjiCdLT0xk0aBBHjx7l4osvZtWqVWzZsoWLLrqIyy67jKysrCav88gjj3DVVVfx7bffcvHFFzNt2jQOHz7crDrY7XYuv/xyDh8+zKeffkpaWho//vgjU6ZMcZaZNm0aPXv2ZMOGDWzatIn77rvPuU7NrFmzqKio4LPPPmPbtm08+eST9Vql3EHdUiIi0vGUloKrv0DPOad55Y4eBReNeXn00Ue54IILnM+7dOnC4MGDnc8fe+wxlixZwrJly7j99tsbvc6MGTOYOnUqAI8//jjPPPMMX3/9NRdddNFJ67Bq1Sq2bdvG7t27iYuLA+DVV19lwIABbNiwgTPPPJOsrCx+97vfkZSUBEC/fv2cr8/KymLy5MkMHDgQgD59+rTgDrSOWm5ERETaqOHDh9d7fvToUe655x6Sk5MJDw8nODiY9PT0k7bcDBo0yPn3oKAgQkNDOXDgQLPqkJ6eTlxcnDPYAKSkpBAeHk56ejoAd911FzfccAPjxo3jiSee4IcffnCWnT17Nn/60584++yzeeihh/j222+b9b6nQuFGREQ6nsBARwtKcx5r1zbvmmvXNu96Ltz+4fhZT/fccw9Llizh8ccf5/PPP2fr1q0MHDjwpDuhH7+VgWEYLt1g9OGHH2b79u1ccsklrF69mpSUFJYsWQLADTfcwI8//sg111zDtm3bGD58OP/85z9d9t4NUbg5FZGRjhH0TfH3d5QTERHPMQxH11BzHgEBzbtmQEDzrufGVZK/+OILZsyYwaRJkxg4cCAxMTHs2bPHbe8HkJycTHZ2NtnZ2c5jO3bsoKCggJSUFOex/v3789vf/paVK1dyxRVXsGDBAue5uLg4brnlFt59913uvvtuXnrpJbfWWWNuTkV8vGNqYH4+9uoqssafRbEfhLzyOvExiVgMi1YoFhERl+nXrx/vvvsul112GYZh8MADD7i0BaYh48aNY+DAgUybNo358+dTXV3Nbbfdxnnnncfw4cMpKyvjd7/7Hb/61a/o3bs3+/btY8OGDUyePBmAOXPmMGHCBPr378+RI0dYs2YNycnJbq2zws2pio8nPaCEJRkfkTHWQrnFjv+Rd0kKHcakpEkkRynYiIi0abWt8Cdb56YNtML/7W9/47rrrmPUqFFERkby+9//nqKiIre+p2EYvPfee9xxxx2MHj0ai8XCRRdd5OxaslqtHDp0iGuvvZa8vDwiIyO54ooreOSRRwDHJpizZs1i3759hIaGctFFF/H3v//dvXU2TRfOWWsHioqKCAsLo7CwkNDQ0FO+XvrBdJ5Z/wz5pfnE/XcZQcWVlNw4nWxrCZGBkcweOZvkKPcmVBGRzq68vJzdu3fTu3dv/E82XKAh2iewTWjq59iS72+13JwCu2lnScYS8kvzSY5MpoYPqTEqCcVGSlQvdhzcwdKMpSRGHuuiEhGRtik+XuGlA9E37inIKswiIz+DuLA4iiuL+aJHDVtigaoqDMOgZ2hP0vPTySpseoqeiIiIuI7CzSkoriimvLqcIN8gxwZoFgt2A6iqAiDIL4jy6nKKK4q9W1EREZFOROHmFITYQvD38aekqsTR7WS1YtYJNyWVJfj7+BNiC/FuRUVERDoRhZtTEB8WT1JkEtmF2RjUabmprsY0TfYV7SM5Mpn4MPXjioiIeIrCzSmwGBYmJU0iMjCS7w9/T7kVqoHCiiJ2HNxBZGAkE5MmajCxiIiIB+lb9xQlRyUze+RshkQPoczH5GAwHKosYGjsUE0DFxER8QJNBXeB5Khk7j37Xgr/9zYVe4u4p+f59D7n92qxERER8QKFGxfxtfoSbg2CEoir8FOwERER8RJ9A7uIj8UHju26Wl1W4uXaiIhIZzFmzBjmzJnjfJ6QkMD8+fObfI1hGCxdurTZ12xvFG5cxGJYMHwc4aZG4UZEpF2ym3b2FOxhW9429hTswW66b1PKyy67jIsuuqjBc59//jmGYfDtt9+2+LobNmzgpptuOtXqtWvqlnKFY3uS+JRXUAVU78uCzZt/Pq89SURE2rz0g+ksyVhCRn4G5dXl+Pv4kxSZdGwTZNdPDrn++uuZPHky+/bto2fPnvXOLViwgOHDhzNo0KAWXzcqKspVVWy31HJzqrKyIDERhg3Dum49ADUrPoJhw35+JCY6yomISJtUuwnylpwtRAZGktg1kcjASLbkbOGZ9c+QfjDd5e956aWXEhUVxcKFC+sdP3r0KG+//TbXX389hw4dYurUqfTo0YPAwEAGDhzIG2+80eR1j++W2rlzJ6NHj8bf35+UlBTS0tJaXNcjR45w7bXXEhERQWBgIBMmTGDnzp3O83v37uWyyy4jIiKCoKAgBgwYwPLly52vnTZtGlFRUQQEBNCvXz8WLFjQ4jq0hFpuTlV+PpSXA+BzrPWy+vjIWF7uKKfWGxERjzBNkyp7VbPK2k077+x4hwMlB0iOTHZspwME+QaR2DWR9Px0/i/9/7gn4p5mTRbxtfg6r9EUHx8frr32WhYuXMj999/vfM3bb79NTU0NU6dO5ejRowwbNozf//73hIaG8sEHH3DNNddw2mmnMWLEiJN/NrudK664gujoaNavX09hYWGrxtLMmDGDnTt3smzZMkJDQ/n973/PxRdfzI4dO/D19WXWrFlUVlby2WefERQUxI4dOwgODgbggQceYMeOHXz44YdERkaya9cuysrKWlyHllC4cSHrsXBTc/LfaRERcaMqexWPf/54s8oWlBewZvcaAn0DOVR26ITz5dXl7Dq8i/zSfML9w096vT+c+wf8rH7Neu/rrruOp556ik8//ZQxY8YAji6pyZMnExYWRlhYGPfcc4+z/B133MGKFSt46623mhVuPv74YzIyMlixYgXdu3cH4PHHH2fChAnNqh/gDDVffPEFo0aNAuC///0vcXFxLF26lCuvvJKsrCwmT57MwIEDAejTp4/z9VlZWZxxxhkMHz4ccLQsuZu6pVyo0ZYbERFpsyqqK6i2V+Nr9W3wvJ/Vj2p7NRXVFS5/76SkJEaNGsUrr7wCwK5du/j888+5/vrrAaipqeGxxx5j4MCBdOnSheDgYFasWEFWM4c6pKenExcX5ww2AKmpqS2qY3p6Oj4+PowcOdJ5rGvXriQmJpKe7uiumz17Nn/60584++yzeeihh+oNhL711ltZvHgxQ4YM4d577+XLL79s0fu3hlpuXMhqOv6sUbgREfEqX4svfzj3D80qu6dgD8WVxXQN6EqoLfSE84UVhRwuO8xvU39LQnhCs967Ja6//nruuOMOnnvuORYsWMBpp53GeeedB8BTTz3FP/7xD+bPn8/AgQMJCgpizpw5VFZWtug93O2GG25g/PjxfPDBB6xcuZJ58+bx9NNPc8cddzBhwgT27t3L8uXLSUtLY+zYscyaNYu//vWvbquPvoZdSC03IiJtg2EY+Fn9mvXo26UvKVEp7C/ej8WwYLVYnQ+LYSGnOIcBUQPo26Vvs67XnPE2dV111VVYLBYWLVrEq6++ynXXXee8xhdffMHll1/O1VdfzeDBg+nTpw/ff/99s6+dnJxMdnY2OTk5zmNfffVVi+qXnJxMdXU169evdx47dOgQmZmZpKSkOI/FxcVxyy238O6773L33Xfz0ksvOc9FRUUxffp0Xn/9debPn8+LL77Yojq0lL6GXUhjbkRE2p+6myDvOLiDwvJCqu3VFJYXemQT5ODgYKZMmcLcuXPJyclhxowZznP9+vUjLS2NL7/8kvT0dG6++Wby8vKafe1x48bRv39/pk+fzjfffMPnn3/O/fff36L69evXj8svv5wbb7yRtWvX8s0333D11VfTo0cPLr/8cgDmzJnDihUr2L17N5s3b2bNmjUkJzumzz/44IO899577Nq1i+3bt/P+++87z7mLwo0LqeVGRKR9qt0E+YzYMzhUdojvD33PobJDHtsE+frrr+fIkSOMHz++3viYP/7xjwwdOpTx48czZswYYmJimDhxYrOva7FYWLJkCWVlZYwYMYIbbriBP//5zy2u34IFCxg2bBiXXnopqampmKbJ8uXL8T22Mn9NTQ2zZs0iOTmZiy66iP79+/P//t//A8DPz4+5c+cyaNAgRo8ejdVqZfHixS2uQ0sYpmmabn2HNqaoqIiwsDAKCwsJDT2xb7XFate5KS/n9UGwqwtMzIAhuXXK+PtDZqamgouIuEl5eTm7d++md+/e+Pv7t/o6dtNOVmEWxRXFhNhCiA+L116BHtTUz7El398aUHyq4uMdwSU/H58NC2Dxs1R3CYMPVv9cRisUi4i0CxbD0qxBw9K2Kdy4Qnw8xMdjPbAGgBqLAUOHerlSIiIinZPa2lzIx9cGQHVNtZdrIiIi0nkp3LiQj6+jf7C6mUt+i4iIiOsp3LiQ1dex3HaNWm5ERLyik82R6XBc9fPzariZN28eZ555JiEhIXTr1o2JEyeSmZl50te9/fbbJCUl4e/vz8CBA507j3qbs1vKrAH9ByYi4jFWqxWgza3cKy1T+/Or/Xm2llcHFH/66afMmjWLM888k+rqav7whz9w4YUXsmPHDoKCghp8zZdffsnUqVOZN28el156KYsWLWLixIls3ryZ008/3cOfoD6rnyPc1BhAdTX4tmwJbhERaR0fHx8CAwM5ePAgvr6+WCzqmGhv7HY7Bw8eJDAwEB+fU4snbWqdm4MHD9KtWzc+/fRTRo8e3WCZKVOmUFJSwvvvv+88dtZZZzFkyBBeeOGFk76Hy9e5qePT9I9Yc9sEhu2Hy7aUQGCgS68vIiKNq6ysZPfu3djtdm9XRVrJYrHQu3dv/PxO3FW93a5zU1hYCECXLl0aLbNu3TruuuuuesfGjx/P0qVLGyxfUVFBRcXPO7kWFRWdekUb4RxzYwGqNKhYRMST/Pz86Nevn7qm2jE/Pz+XtLq1mXBjt9uZM2cOZ599dpPdS7m5uURHR9c7Fh0dTW5uboPl582bxyOPPOLSujbG4utHgT/8EAF78ncRH3qGVrYUEfEgi8VySisUS8fQZsLNrFmz+O6771i7dq1Lrzt37tx6LT1FRUXExcW59D0A0g+m8/p3i/giAQKq4Juv5pG0fyiTkia5fU8SERER+VmbCDe3334777//Pp999hk9e/ZssmxMTMwJO6Lm5eURExPTYHmbzYbNZnNZXRuSfjCdZ9Y/w87DOwmsgehiiPQNY0vOFrILsz2y6ZqIiIg4eLXPxDRNbr/9dpYsWcLq1avp3bv3SV+TmprKqlWr6h1LS0sjNTXVXdVskt20syRjCfml+fSN6IvNbsEwINQaSEpUCvml+SzNWIrd1AA3ERERT/BquJk1axavv/46ixYtIiQkhNzcXHJzcykrK3OWufbaa5k7d67z+Z133slHH33E008/TUZGBg8//DAbN27k9ttv98ZHIKswi4z8DOLC4rBarGBYsBuA3Y5hGPQM7Ul6fjpZhVleqZ+IiEhn49Vw8/zzz1NYWMiYMWOIjY11Pt58801nmaysLHJycpzPR40axaJFi3jxxRcZPHgw77zzDkuXLvXaGjfFFcWUV5cT5BuEYRhgMTANoKYGgCC/IMqryymuKPZK/URERDobr465ac4SO5988skJx6688kquvPJKN9So5UJsIfj7+FNSVYKBAYaBCc5wU1JZgr+PPyG2EK/WU0REpLPQPOVTFB8WT1JkEtmF2Y4DhsXRcmO3Y5om+4r2kRyZTHxYvFfrKSIi0lko3Jwii2FhUtIkIgMj2XV4F+U+JtUGFFYUsePgDiIDI5mYNFHr3YiIiHiIvnFdIDkqmdkjZzOw20DKfOBAMByqKGBo7FBNAxcREfGwNrHOTUeQHJXMrBGzqHx9If77D3LL+N8Qf850tdiIiIh4mMKNC/lYfAg3bYSXQIIRAQo2IiIiHqdvXxeyGBawHlvnRhtnioiIeIXCjQtZDAtYrNQo3IiIiHiNwo0LOVpurI6Wm8pKb1dHRESkU1K4cSF1S4mIiHifwo0L1XZLKdyIiIh4j8KNC6lbSkRExPsUblxI3VIiIiLep3DjQrXdUqYBplpuREREvELhxoVqu6UA7FUVXq6NiIhI56Rw40JWixWsjltqr1LLjYiIiDco3LhQ/ZYbhRsRERFvULhxodoxNwB2jbkRERHxCm2c6SpZWRgHD0JpKQD2nP2wefPP5yMjIT7eS5UTERHpPBRuXCErCxITMcrLsZwHdgPsS5fAW0t+LuPvD5mZCjgiIiJupm4pV8jPh/JyACym45DdOK5MebmjnIiIiLiVwo2LNRpuRERExCMUblysNtzU6M6KiIh4hb6CXUwtNyIiIt6lcONiCjciIiLepXDjYgo3IiIi3qVw42IKNyIiIt6lcOMKkZGOdWxoItz4+zvKiYiIiFtpET9XiI93LNCXn49lxWOwcin2lGR49vWfy2iFYhEREY9QuHGV+HiIj8eyNQYAu58vDB3q5UqJiIh0PuqWcjGr1ZEX7fZqL9dERESkc1K4cTGL1RcAe02Nl2siIiLSOSncuJjFagXAXl3l5ZqIiIh0Tgo3Lmap7ZaqUbeUiIiINyjcuJjF51i3lF3dUiIiIt6gcONiarkRERHxLoUbF1O4ERER8S6FGxernS1Vo24pERERr1C4cbGfx9yo5UZERMQbFG5czOKjbikRERFvUrhxMeeYG3VLiYiIeIXCjYtZfPwArVAsIiLiLQo3LubsljLtYJpero2IiEjno3DjYs69pQygWuNuREREPE3hxsWsPgo3IiIi3qRw42IW32NjbhRuREREvELhxsUsdVtuqrQzuIiIiKcp3LiYcyq4Wm5ERES8QuHG1QyDggD4IQL2HNntmDUlIiIiHuPj7Qp0JOkH01n83WI2JMDGGPjq67+QdGAYk5ImkRyV7O3qiYiIdApquXGR9IPpPLP+GXYd3kVgjUH3Yoj0C2NLzhaeWf8M6QfTvV1FERGRTkHhxgXspp0lGUvIL82nd3hvbHYLFhNCrYGkRKWQX5rP0oyl6qISERHxAIUbF8gqzCIjP4O4sDgshgUMA9MA7HYMw6BnaE/S89PJKszydlVFREQ6PIUbFyiuKKa8upwg3yAMw6gXbgCC/IIory6nuKLYuxUVERHpBBRuXCDEFoK/jz8lVSUYHAs34Aw3JZUl+Pv4E2IL8Wo9RUREOgOFGxeID4snKTKJ7MJsx4E6LTemabKvaB/JkcnEh8V7tZ4iIiKdgaaCu4DFsDApaRLZhdnsPLyTcitUG1BYUcS+g4VEBkYyMWmiYzyOiIiIuJW+bV0kOSqZ2SNnkxyZTJmvSU4IHKosYGjsUMdxrXMjIiLiEWq5caHkqGRuGHoD1jfeIGrvIab98hriz5mmFhsREREPUrhxMR+LD+E1fvQohgSfSFCwERER8Sh987qYxbCAxaJdwUVERLxE4cbF6oUb7QouIiLicQo3LqZwIyIi4l0KNy5mMSxgVbeUiIiItyjcuJhabkRERLxL4cbFFG5ERES8y6vh5rPPPuOyyy6je/fuGIbB0qVLmyz/ySefYBjGCY/c3FzPVLgZHOHGqm4pERERL/FquCkpKWHw4ME899xzLXpdZmYmOTk5zke3bt3cVMOWU8uNiIiId3l1Eb8JEyYwYcKEFr+uW7duhIeHu75CLlAbbmoUbkRERLyiXY65GTJkCLGxsVxwwQV88cUXTZatqKigqKio3sOd1HIjIiLiXe0q3MTGxvLCCy/wf//3f/zf//0fcXFxjBkzhs2bNzf6mnnz5hEWFuZ8xMXFubWOmgouIiLiXe1qb6nExEQSExOdz0eNGsUPP/zA3//+d1577bUGXzN37lzuuusu5/OioiK3Bhy13IiIiHhXuwo3DRkxYgRr165t9LzNZsNms3msPgo3IiIi3tWuuqUasnXrVmJjY71dDSerxaqp4CIiIl7k1Zabo0ePsmvXLufz3bt3s3XrVrp06UJ8fDxz587lp59+4tVXXwVg/vz59O7dmwEDBlBeXs7LL7/M6tWrWblypbc+wgnUciMiIuJdXg03Gzdu5Pzzz3c+rx0bM336dBYuXEhOTg5ZWVnO85WVldx999389NNPBAYGMmjQID7++ON61/CqrCwsudlQWoppgPnTPoy6g50jIyE+3nv1ExER6QQM0zRNb1fCk4qKiggLC6OwsJDQ0FDXXTgrCxITKa8u54lzHIce+BSsde+uvz9kZirgiIiItFBLvr/b/ZibNiM/H8rLsdQJM3bjuDLl5Y5yIiIi4jYKNy7WZLgRERERt1O4cTGFGxEREe9SuHExQ+FGRETEqxRuXMzg59YbhRsRERHPU7hxA4UbERER71G4cYPacFOjuysiIuJx+vp1lchIxzo2NNFy4+/vKCciIiJuo3DjKvHxjgX6Nm3CcsEFANiHD4NNm35+aAE/ERERt2v3u4K3KfHxEB+PZV0UAPYAfxg61MuVEhER6VzUcuMGVqsvAPZq7QouIiLiaQo3bmCxWgGw12hXcBEREU9TuHEDi9XR26dwIyIi4nkKN25gsdSGG3VLiYiIeJrCjRtYfGrDTY2XayIiItL5KNy4gbqlREREvEfhxg0UbkRERLxH4cYNLLVTwRVuREREPE7hxg3UciMiIuI9Cjdu4Aw3dg0oFhER8TSFGzdwzpbSCsUiIiIep3DjBs4xN2q5ERER8TiFGzeobbmpUbgRERHxOIUbN/h5zI0GFIuIiHiawo0bWH1tgGZLiYiIeIPCjRs4W24A7Hav1kVERKSzUbhxA4vPsQHFBlCt1hsRERFPUrhxA2fLjQFUaTq4iIiIJyncuIHF1w9Qy42IiIg3KNy4g9VCgT/sioA9h3/EbmrcjYiIiKf4eLsCHU36wXTeyXiXLQmwJQY+Xfc4SfvOYFLSJJKjkr1dPRERkQ5PLTculH4wnWfWP8MPh38ksBp6FEGkXzhbcrbwzPpnSD+Y7u0qioiIdHgKNy5iN+0syVhCfmk+CeEJ2OwGFhNCfYJIiUohvzSfpRlL1UUlIiLiZgo3LpJVmEVGfgZxYXFYDAsYBqYB2O0YhkHP0J6k56eTVZjl7aqKiIh0aAo3LlJcUUx5dTlBvkEYhvFzuKlx7C8V5BdEeXU5xRXF3q2oiIhIB9eqcJOdnc2+ffucz7/++mvmzJnDiy++6LKKtTchthD8ffwpqSrBwBFu6q5QXFJZgr+PPyG2EK/WU0REpKNrVbj5zW9+w5o1awDIzc3lggsu4Ouvv+b+++/n0UcfdWkF24v4sHiSIpPILsx2hpvabinTNNlXtI/kyGTiw+K9XVUREZEOrVXh5rvvvmPEiBEAvPXWW5x++ul8+eWX/Pe//2XhwoWurF+7YTEsTEqaRGRgJHsK91BuhWoLFFYUsePgDiIDI5mYNNExHkdERETcplXftFVVVdhsjp2vP/74Y375y18CkJSURE5Ojutq184kRyUze+RskiKTKPOF/SFwqKKAobFDmT1ytta5ERER8YBWhZsBAwbwwgsv8Pnnn5OWlsZFF10EwP79++natatLK9jeJEclc90Z13H+gSAmpsOjCTP4/Tm/V7ARERHxkFaFmyeffJJ//etfjBkzhqlTpzJ48GAAli1b5uyu6sx8Lb6E1/jS4ygk+EapK0pERMSDWrX9wpgxY8jPz6eoqIiIiAjn8ZtuuonAwECXVa69shgWsFio0a7gIiIiHteqJoWysjIqKiqcwWbv3r3Mnz+fzMxMunXr5tIKtkdWixWsFu0KLiIi4gWtCjeXX345r776KgAFBQWMHDmSp59+mokTJ/L888+7tILtkbPlxoLCjYiIiIe1Ktxs3ryZc889F4B33nmH6Oho9u7dy6uvvsozzzzj0gq2R1bDqm4pERERL2lVuCktLSUkxLHS7sqVK7niiiuwWCycddZZ7N2716UVbI+sFitYrOqWEhER8YJWhZu+ffuydOlSsrOzWbFiBRdeeCEABw4cIDQ01KUVbI/ULSUiIuI9rQo3Dz74IPfccw8JCQmMGDGC1NRUwNGKc8YZZ7i0gu2RuqVERES8p1VTwX/1q19xzjnnkJOT41zjBmDs2LFMmjTJZZVrrxzdUpotJSIi4g2tCjcAMTExxMTEOHcH79mzpxbwy8qC/Hws5flQWenolvrxR9i82XE+MhLitXGmiIiIO7WqW8put/Poo48SFhZGr1696NWrF+Hh4Tz22GPY7XZX17F9yMqCxEQYNgzrBeNhzx5Ht9QTT8CwYY5HYqKjnIiIiLhNq1pu7r//fv7973/zxBNPcPbZZwOwdu1aHn74YcrLy/nzn//s0kq2C/n5UF4OgNV0HLIbx5UpL3eUU+uNiIiI27Qq3PznP//h5Zdfdu4GDjBo0CB69OjBbbfd1jnDTR2WY+GmRltKiYiIeFyrvn4PHz5MUlLSCceTkpI4fPjwKVeqvbMe65mrOb7lRkRERNyuVeFm8ODBPPvssyccf/bZZxk0aNApV6q9q+2WMo0GuqZERETErVrVLfWXv/yFSy65hI8//ti5xs26devIzs5m+fLlLq1ge1TbLQWOcFP3uYiIiLhXq1puzjvvPL7//nsmTZpEQUEBBQUFXHHFFWzfvp3XXnvN1XVsd6x1Joypa0pERMSzWr3OTffu3U8YOPzNN9/w73//mxdffPGUK9aeWY9ruRERERHP0XweV4mMBH9/AIw64abejCl/f0c5ERERcRuFG1eJj4fMTNi0CWPTJqwjzgKgZvwFsGmT45GZqTVuRERE3KzV3VLSgPh4Z3ixpnWlBrAHB8PQod6tl4iISCfSonBzxRVXNHm+oKDgVOrSoVh8fAGoqSz3ck1EREQ6lxaFm7CwsJOev/baa0+pQh2F1RluKrxcExERkc6lReFmwYIF7qpHh2O1OsKNvarSyzURERHpXDSg2E0svn4A1FSp5UZERMSTvBpuPvvsMy677DK6d++OYRgsXbr0pK/55JNPGDp0KDabjb59+7Jw4UK317M1rD614UYtNyIiIp7k1XBTUlLC4MGDee6555pVfvfu3VxyySWcf/75bN26lTlz5nDDDTewYsUKN9e05WrH3NjVciMiIuJRXp0KPmHCBCZMmNDs8i+88AK9e/fm6aefBiA5OZm1a9fy97//nfHjx7urmq3i7JaqrvJyTURERDqXdjXmZt26dYwbN67esfHjx7Nu3bpGX1NRUUFRUVG9hydYNeZGRETEK9pVuMnNzSU6OrresejoaIqKiigrK2vwNfPmzSMsLMz5iIuL80RVnWNuNFtKRETEs9pVuGmNuXPnUlhY6HxkZ2d75H3VLSUiIuId7Wr7hZiYGPLy8uody8vLIzQ0lICAgAZfY7PZsNlsnqhePT93S6nlRkRExJPaVctNamoqq1atqncsLS2N1NRUL9WoYXbTzmF7CXlBkB1Qid20e7tKIiIinYZXW26OHj3Krl27nM93797N1q1b6dKlC/Hx8cydO5effvqJV199FYBbbrmFZ599lnvvvZfrrruO1atX89Zbb/HBBx946yOcIP1gOksylvBR1goO9YKssCq+/fwJJiVPIjkq2dvVExER6fC8Gm42btzI+eef73x+1113ATB9+nQWLlxITk4OWVlZzvO9e/fmgw8+4Le//S3/+Mc/6NmzJy+//HKbmQaefjCdZ9Y/Q35pPiG2UCiF0HLYsn8T2UXZzB45WwFHRETEzbwabsaMGYNpmo2eb2j14TFjxrBlyxY31qp17KadJRlLyC/NJyUqhYzc7ygBAquhZ0R/dhTuYmnGUhIjE7EY7ao3UEREpF3Rt6yLZBVmkZGfQVxYHIZhYFitAJgGGHY7PUN7kp6fTlZh1kmuJCIiIqdC4cZFiiuKKa8uJ8g3CACLxRFu7AZQU0OQXxDl1eUUVxR7sZYiIiIdn8KNi4TYQvD38aekqgQ4Fm4shiPcVFdTUlmCv48/IbYQ71ZURESkg1O4cZH4sHiSIpPILszGNE3HuBrDEW7M6mr2Fe0jOTKZ+LB4b1dVRESkQ1O4cRGLYWFS0iQiAyPZcXAHZVVl2C0GxX6wo+B7IgMjmZg0UYOJRURE3EzftC6UHJXM7JGzOSP2DIorizkUAEU2GBrcX9PARUREPKRdbb/QHiRHJZMYmUhyZDIrl3/J6TuLufWmqVgUbERERDxCLTduYDEsxIfFE11lo0s5WCq1eaaIiIinqOXG1bKyID8fn8KdYLdTbQG2b4fQUMf5yEiI16BiERERd1G4caWsLEhMhPJyfKKAATjCze23/1zG3x8yMxVwRERE3ETdUq6Unw/l5QD4HtsIvPr4O1xe7ignIiIibqFw4yY+x8JNldW79RAREelsFG7cxKexlhsRERFxK331uonCjYiIiHfoq9dNfGscfyrciIiIeJa+et3EOeZGd1hERMSj9NXrJnW7pUzvVkVERKRTUbhxpchIxzo2/BxuTAPsRp0y/v6OciIiIuIWCjeuFB/vWKBv0yZ81n0FI0YAUH3pBNi0yfHQAn4iIiJupRWKXS0+HuLj8TFNWN0VgGp/G7ahQ71cMRERkc5BLTduYhgGPn6OLqqq8hIv10ZERKTzULhxo9pwU12mcCMiIuIpCjdu5Aw35aVeromIiEjnoXDjJnbTTpG1mrwg2E0BdtPu7SqJiIh0ChpQ7AbpB9NZkrGEFQWbONoL8krzGL72CSYlTSI5Ktnb1RMREenQFG5cLP1gOs+sf4b80nyCbSHYSiGszM6WnC1kF2Yze+RsBRwRERE3UreUC9lNO0sylpBfmk9KVApBfsFYgMCyalKiUsgvzWdpxlJ1UYmIiLiRwo0LZRVmkZGfQVxYHIZhYPX1BcBeU4MB9AztSXp+OlmFWd6tqIiISAemcONCxRXFlFeXE+QbBIDFxw84tv1CdTVBfkGUV5dTXFHsxVqKiIh0bBpz40IhthD8ffwpOZRDaLUV61HH+jY1FiA7mxKfGvwrSwnJL4Jo79ZVRESko1K4caH4sHiSfGLY8tbTpOTa8ekKhEC1AeZrr7EvCobmQPwdH0Pm99pjSkRExA3ULeVCFsPCpNCRRBbb2REF5VawA0U22BEFkaUwMQMs5RWQn+/t6oqIiHRICjculhzSm9nr4YwcKPGDQ4FQ4O9osZm9HpKVaURERNxK3VJukJwPiV84/lx5GgzMg1s2gcX0ds1EREQ6PrXcuInFhIQCiC6BsAoFGxEREU9RuHEjW43jzwq1j4mIiHiMwo0b2aodf1ZYvVsPERGRzkThxo3UciMiIuJ5CjeuFhkJ/v7Azy03lce33Pj7O8qJiIiIyyncuFp8PGRmwqZN2FZ/CmeeSYUVzMsuhU2bHI/MTC3gJyIi4ibqMHGH+HiIj8e3qoyCT4OpCIEfAsrpc8YQLIbypIiIiDsp3LhJ+sF0lqQvYU3ND1T3gkL7N5y+9gkmJU0iOSrZ29UTERHpsBRu3CD9YDrPrH+G/NJ8gv2CsJZCeE01W3K2kF2YzeyRsxVwRERE3ER9JC5mN+0syVhCfn4WKdURBFVbsAABRytIqY4gf/9Olq5+DvvePd6uqoiISIeklhsXyyrMImPvJuLe/gij1I5PD6j0hZqyUoyXXqKnDdIDVpF150skfL1TA4tFRERcTC03LlZcUUx5aRFBZXYAfBx/UH3sTgdVQrkPFJuV2hlcRETEDRRuXCzEFoK/xUaJn+O577GF/KqO3ekSP/CvhpBK79RPRESko1O4cbH4sHiSghPIDgUT8D3WclNpdTzfF+rYLTy+0Ju1FBER6bgUblzMYliYFHs+kaWwIwoqLWAHCmyO55GlMDFDu4SLiIi4i8KNGySH9Gb2ejgjB0p94VAgHAmAoTkwe72j5UZERETcQ7Ol3CQ5HxK/gMF58H5/SMyH2V+rxUZERMTd1HLjRhYTTjvs2B38SABkhYHd8HatREREOja13LjDsZ3B04PLWTgY1iSAYcLucEjKh0kZkHxUO4OLiIi4g8KNO8THk77+fZ5Z+zT7S/IIPJyF//58IkOi2HJ2b7KvCGZ23JVoAwYRERHXU7hxA7tpZ8neFeR/voLT8+x80QtMCwTtPUhK9kF2RMHSd1eTuNGGJfN7rVIsIiLiQhpz4wZZhVlk5O0g7ogdH9PRJQWOhfwMoGcRpEdCln+FVikWERFxMYUbNyiuKKbcXkFQpSPM+NWuUmx1/OncgsHPa1UUERHpsBRu3OD4LRhsx8JN+bFOQG3BICIi4j4KN25w/BYMtmrH8QptwSAiIuJ2CjducPwWDFUGlPo4poJv6A5dtQWDiIiI2yjcuEntFgw9CmFHN8jsCt/GQE6wo0vKKSfHa3UUERHpiBRu3KzcF2KOQq8CGJgLY/bCT6HwzEjHjCmuuAKysrxdTRERkQ5D4cZN7F27sGSAhUOBMCwHulSArwlhFZByEPIDYWkS2KsqNR1cRETEhRRu3CQrDDImn0dc0c/dUJXHBhTXW+smzJu1FBER6XgUbtykuKKYcpuVoErHOjeGCabx83RwrXUjIiLiHm0i3Dz33HMkJCTg7+/PyJEj+frrrxstu3DhQgzDqPfw9/f3YG2bp+5aNwYQUOUIM1mhUOAPR+uudaNBxSIiIi7j9b2l3nzzTe666y5eeOEFRo4cyfz58xk/fjyZmZl069atwdeEhoaSmZnpfG4Yhqeq22y1a91sCYWoEsgOhQPBkBsEoRVgBy748dhaN1dcATt3ao8pERERF/B6y83f/vY3brzxRmbOnElKSgovvPACgYGBvPLKK42+xjAMYmJinI/o6GgP1rh5ate6sdphRV8oCACfGvCtgTIfKPWDnBDHFHEqNahYRETEVbwabiorK9m0aRPjxo1zHrNYLIwbN45169Y1+rqjR4/Sq1cv4uLiuPzyy9m+fXujZSsqKigqKqr38JTE4F7EHHV0SQVWQuWxUBNfBBftghrLsRlTba/hSUREpN3yarjJz8+npqbmhJaX6OhocnNzG3xNYmIir7zyCu+99x6vv/46drudUaNGsW/fvgbLz5s3j7CwMOcjLi7O5Z+jMVn+FRwONrjgR/jFHsdaN3GFMCobupUeN2NK425ERERcwuvdUi2VmprKtddey5AhQzjvvPN49913iYqK4l//+leD5efOnUthYaHzkZ2d7bG6FkeGUn7uKIIroVuJY40bDEeYKfB3tOY4Z0xpMT8RERGX8OqA4sjISKxWK3l5efWO5+XlERMT06xr+Pr6csYZZ7Br164Gz9tsNmw22ynXtTVCbCH4B4VT4ufYNHN/MBwOhIOBEFwJwRUQWXpsxlTtuBsNKhYRETklXm258fPzY9iwYaxatcp5zG63s2rVKlJTU5t1jZqaGrZt20ZsbKy7qtlqtTOmtkfC+h5Q4ucYUBxQ6RiHszcccoOhxPfYC9Q1JSIicsq83i1111138dJLL/Gf//yH9PR0br31VkpKSpg5cyYA1157LXPnznWWf/TRR1m5ciU//vgjmzdv5uqrr2bv3r3ccMMN3voIjbIYFi6POY8Cf0eICa1w7ARebIMDgRBe5uiqWpZ4bFCxuqZEREROmdfXuZkyZQoHDx7kwQcfJDc3lyFDhvDRRx85BxlnZWVhsfycwY4cOcKNN95Ibm4uERERDBs2jC+//JKUlBRvfYQmBUV1J6bMwMdukhMMhwLAboGQY3tNlfjCVz0c43ASCtQ1JSIicqoM0zRNb1fCk4qKiggLC6OwsJDQ0FC3v9+2vG08tvhWuqR9wdo4x47g/jXQ6wgYhmNgcaUF/rYSLs8E3n8fLrnE7fUSERFpT1ry/e31bqmOLsQWgi0wjG3dHC02UaWOvaaO2qDKArYqx59fxKlrSkRExBUUbtwsPiyeGP+u7Av7ecxNoc2xHcPecNgd4dh7KqN2vRutViwiInJKFG7czGJYOCfhXHzsjkHE+QFQaYUaA6oM8LE7Wmy+i4Jvatcy1KwpERGRVlO48YDBAy9gQMIIaixQZQUTx1YMlT6OYFPm61j/Zln/Y11TjazZIyIiIiencOMB8WHxJEYmggHdj4KtBix2x5o3tmrHmBuLHT7vBasTgN/9TuNuREREWknhxgNqu6asdke3lB0wDccmmocDHd1U/tVQZIP/JYK9ugq2bfN2tUVERNolhRsPGTzwAk6L6ke14eiOqrE4Qo5PDQRWOVYvLrfC9tqBxRMnQhM7o4uIiEjDFG48JD4snqTABOwWx9YLQZXgY0KN1THmptzHMR5nbxgU+gHV1TB6tAKOiIhICynceIjFsDCy6yBMA6otjocdx6PGcEwRN4DcENhWO2tKAUdERKTFFG48aGD8cLoddcyIqjl252ssjgHFdgMqLI7xNx+edmzWFCjgiIiItJDCjQeFdYujd5FjhWLfGscaN4bdca7S6gg61RbHoOJ/D67zQgUcERGRZlO48aD4lFSSR16Gbw1YTUd3FIZj5hQc+9OEo37wh3GwtH+dFyvgiIiINIvCjQdZDAu/PGMKQdVQbYCvHax2MOpsXWoajkX+DgXAjZc1EHDOPRcWLtQ6OCIiIo1QuPGwX5w1lbNPGwPHBhY7x9YcY9ZpzckPhJmXw4tn1ClXUwMzZ0K/fgo4IiIiDVC48TCLYWHG8BvoWuaYIWU1HVPCjQYLQ0EA3HExTL7SsQaOU2UlvPKKAo6IiMhxFG684BdnTWVsyiXOTTOrG0w2OPqncAw2XtYfLpsKS/vVOf/II3DaaeqmEhERqUPhxgsshoW7L3+SPpF9MI613hgmzjDDsa4pJwPsVtgdAb/5FTx87nFTxWfOVMgRERE5RuHGSwZ0G8Bjl/ydLkFdqDGOhZUGQk09BpT5wSNjYNR1sLK3Qo6IiMjxFG686JdJv+SFSf+mq3/YzzOmzOMKNdRlZYH1PWDSr+GCqxsJOX36wF//qpAjIiKdjmGa5vFfpx1aUVERYWFhFBYWEhoa6u3qALA0Yyk3vjuDQ+WFzjVvGh5h3ADTsVfVkFyYvhUu2A0JhY7BygBYLPDAA3DmmTBwIMTHu/4DiIiIuFlLvr8VbtqIpRlLueXd6zhYdgS7heaHmzqMGuhaBmdnw20bYNyeOiEHwGqFP/5RQUdERNodhZsmtNVwA7Ascxm/e+92dhVnY7e24IUm9cOQCbYqOCMXLs2EoXmQeOi4Fp3aoDN+PKSmuu5DiIiIuIHCTRPacrgB2H5gO/e/dQv/y1vbsoDTmGNBp0spnFYA5+92tOiMynasrwPAnXfCBReoNUdERNoshZsmtPVwA2A37Tz22k08nvlvKn1oVRdVPXWnmJvgVw09C+HqbTD92zotOhYL3HADpKRA374KOyIi0mYo3DShPYSbWs+teIyHvnqSQ/YSx4FTCTnHd10dO9a1xNF9NXo3xB2F6JI6XVhGnbAzYoS6r0RExGsUbprQnsINwLa8bfxu2e18tv9LyuzVjoOn2pJzvGO/AZZjrTqh5dCnwDFeZ8oO6FPbsqPuKxER8RKFmya0t3ADjm6qj3/8mCc/foR1uRsos1c5Trgq5NS26tTtvjp2PKgSBubBxZl1WnYOQ8KVN2JJGaDuKxER8QiFmya0x3BTq37I+frnlhxwfdBp4LjFBFs1hB8/OHkf+Nxwk6Pb6he/UNARERGXU7hpQnsON7WcIef9+1l3ZKsj5DSwVYPL1f1NOW5w8rRtjqATVgFhV0wlvv+ZWPr1V6uOiIi4hMJNEzpCuKlVG3Ke+/SvfLn/K45UH6WmoR+nK4PO8V1XdY77VUNkGfQscgSeofuPrbEz/EISep6OJS5eA5NFRKRVFG6a0JHCTS27aWdPwR5W7lrJu1v/S/rBdPIrC6mwV/+8nUMtd7To1HXcb5NvNXQpg4EH6szIOnM0iSmjSeg/AsugwWrZERGRk1K4aUJHDDd11QadjIMZbMlYzadb32Nz8S4OB3Bi0AH3zLwyaHTqee24nYhSSMmHMXtg6GmpJHZLIaHHAHVliYhIgxRumtDRw83x7KadPdu/YMX6N1iU/T7pJdkU+ENNQ/vBu7tVB07c9RzwrYLwCuh/CC7aCcMOQOJlM0j4xRVq2REREUDhpkmdLdzUVRt0Vn69mHd/+pj0in0crimlwsfDYef4gcnHnfOthqgSR8vO6D0Ql3gG0ZEJJEYPUFeWiEgnpXDThM4cbupydl/t+Iy8nVv5Kf1rlh9Yx7fRUOJHw8HGXTOwGpl67nzb2sUFj7XuXLgL4hPPIDoumcSB55MwfByWXgluqJyIiLQVCjdNULhpnH3vHn7csJI3v3uDD7I/YXcYFNnwfMtOrcbG75iOwONfZ+zOaEsvYoOiKQi3ERgezenxwxiVehU+CX3cXEkREfEEhZsmKNw0j33vHvZs/JiMHzeQty+dnzZ9zmcJsDkWzw1OPplGfnONY4OWY4vg0vLuDA3uS1T3RELi+hCWPISwhCTiw+Ide2eJiEi7oHDTBIWbVsrKwv5xGns2pLFiw5ssOh3So6DQH2qMBsKOp4LOcTueN/i+dvA9NpYnrAK6GYFEWoIZ7NuTAaH9iI1LInHwOBIGjFLgERFpoxRumqBw4wJZWdi//YY9mV+zMmsNH+d8QXYI/BTqaNUp92mkZQc837pTq7FBzCbYqqBrKaRYuzLaL5HY0DgKogIJjOvD6UmjGRU3Ch+Lj6drLCIidSjcNEHhxvXse/eQtWk1hbszKNy+ie3frObdZNjeDY4EQJXF0brjsUHKjWlsdeW65+owTLDZIbbKxqXlvRgSkQi9ekPPHtA1Ertp53DZYSpqKugZ1pNz48+lT0Qftf6IiLiBwk0TFG48oE7LTsbuDeStXYFpwv4Q+CzB0Z11IBAqG2oM8VbLDjS9AGHt+eMdK2cBAg0/Tg/ozcX9L6VHfIrjuGEhOjiaxMhEEsITFHxERFpJ4aYJCjdekJUF27bBrl3Ys7PYs3MDGds/Z8VpsLyfozurTXZlNeT4/1qaCEG1p/ywEGoJoH9gHBf2m0DP+NPrFbebdgrKCwj0C+T0bqerG0xEpAEKN01QuGkj1q2Dr7+mevs2vlzxb7ZFwY8R8E00ZEbBYX9H4LE31tDRlgJPQ5oY41NPne46AwObxUZsSCyX9r+UITFD6hWtDUH+vv5EBUbRv2t/IgIiNPNLRDoFhZsmKNy0Qce37GRvI2NTGnlBsCUGPjzWulPhA/aThZq2Fnqa28XVVJkGxitZsBDoG0hCWAL9u/ZnSMwQeoT2cI4DKqsuc4SggChiQ2LVLSYi7Z7CTRMUbtqJrCxYtQq2b6f6x118uek9tkU5Vk+usMDnCY6xO4f9HWN3qhsbsAxNj6Np65oTgk5yzmax0TWwKylRKYyOH91gCOrq3/WE4KPxQiLSlijcNEHhpp2q07pDdjb2vFz25O8iY+d68oIcGeCnEFjZF3Z2daysXGltZGXlutpj4GlIU/t1Ha8Zn9k4VsjP4keoLZT+XfpzYd8L6Rnas165pkKSxbAQFRRFkF8QRyuPcrDkIAaGApOItIrCTRMUbjqYOi085OVh/2kfe7Z8QkZXyAtyBJ6tsZAdCnvC4Yg/VDVnrG5HCT111Z0NBs37jK28D7XhyIrV+Z6mYWKaJn5WP7oEdKnXkgRNByVQS5JIZ6dw0wSFm07g+FaeH38ga81SCv0c4eaHCEfoKfSvP4D5pN1bDWlsrExHCEetGS/kps9tHPufzWoj3D+cPhF9OCP6DHpG9GywS6053W4NlfGx+ChAibRRCjdNULjppI4LPOTlwU8/Yf9kDXvCcLb0NNS9VWVtZIuJ5ugIIae1mhvy2tA9smDBZrURERDhbFmKDYltVVBqbZiym3ayCrMorigmxBai2XAixyjcNEHhRuqpG3q2b4effoIDB7Bv2nhC6DmeHceg5i/jHV1eRTZH689Jg1DdrqG6x6Tl2vl9Oz5MJXdN5qfin9hfvJ8qexU+Fh9iQ2IZGjPU2X0HzQtTzS3XnNAFKHCJ1yncNEHhRpqloZaew4fhwAHYuLFeUbtBg0Foa0unsTdHU9foKN1hnqT71aC6oatXWC+CfB2DwpsKXND6MNWaMq66Vu1Yrn5d+2ExLJRUljgDHLQ81KnlzX0UbpqgcCOnrLHgU1gIhw5BRoazaLUBX8bhnMbepcSxVUNdLu0Gg5MHoJOVkZbT/WyXasdy+Vp88fPxI9A3kK7+XQmxhRDsF4zFsFBtr6bGrMFiWIiwRdA7vDehAaENBqX9xfvZkrPlpC1v0D6DYHPLuWvwv8JNExRuxO2Om8HVWPCpq7HWn7p+CnF0ge0LhYOBcLR2uvuxINTiMHSywcL6wvYe3fs2xcDAbPC/SjmegYHFsBDgE0DP0J6M7zuem4fdTHJU8ilfW+GmCQo34lUNjfEpLITiYigrg9JSx7FG2A3ICoNCP8dsryI/R9Cp/Y/YDhwOhDIfKGpgNtgptQhB88cLNTTtvKnZV/oy9yzdb3EzH8MHm4+NCP8Izo4/m4fOe+iUA05Lvr+1O5+IJ8XHOx5NOb7b68cfHWN9iouxlJWRcJIAVFdjLUJ1Q5B/DXQ9rrus0fFCDf1TqLF/HpmcGHKaKgsn34ZCXMOT/6TVz63TqV3nym7aKa8u57sD37EkYwmJkYkeG3+kcCPS1pxiAKKsDCoroboay08/0acA+hS0sA7fQnXayccLNRaS9h9bPPFwABT7Qf6xMlVWKPOFaksj3WlNfek2dO5km5Ke7Bon++JVq9Kp81bfgH5uXmNiYjftmKZJjb2G8qpyNu7fSFZhlnP2nbsp3Ii0R80JQNC8EARQXX1Ca5CPCaOzHI/WqO1CK/aDoGNvU9xIdxo03Zq0PwQ+S4Bvox1dbM5Vplsahlpb5lQ3P22qTHPeS1quLQ246IQ/U7PO/+ymnZLKEoorij32/go3Ih1Zc0MQ/ByENmyAH374eSD08UEIYO/ek17OYkJCQeuq3RD7l44uth1dYUdU411qcPJut4bKVDa1Iasrg1JzyjSnNas5Yaq55Trhl69HtaWgdTIu+l0w6vzPYlgI8gsixBbimos3g8KNiDjUBqFLLjl52aamwxcX/xyEqqocrULQYOtQS1hMnF1sl/7Q6ss0qTZA1R2j1Jqg1Now5Rzw7aqg1NxyzSnT3O5Dd7ZyuTK8aSxXw1wQxAzAgh2jugprjQV/o4rhgX2dawd5gmZLiYjnNLINBocOndg6VMvX13E8q5X9Y21Yc5YAaPB1nDxMNbdci1qwOouTBczj70VTYaojBcEWlPOxg60aIirg7Cx4aJ0vyet2Nb8luQGaLSUibVNLusmOt24dfP11wy1FZWVgmo5HbUtRXafYauQudVuj2pKGWrCa9Trc08rlyvDW0Fiu2iBX7gN2C2Ae++42wXps1p9pOFrWmtVN6elWNU+/XxPlDBy/1/7V0LMILtoFN22G5PwqyM8/pXDTEgo3ItI+pKY6Hq3V3DFFvr7g41O/S61WGw1JrtZWQ5c7NBXkLEBUCQRVwVE/yAuEQ80IXc16X9pXEGxJOQsQXQKJhyCh0PH75GkKNyLSObRkTFFTTrYQY92uNV/fxluToNOEpbasMwW5zkThRkSkJU6la60hLWlRaioo1ZapbXUqL1dwkk5L4UZExJtc1aLUkJNt8lpW1vhrWxKmSks75IBvab/aRLh57rnneOqpp8jNzWXw4MH885//ZMSIEY2Wf/vtt3nggQfYs2cP/fr148knn+Tiiy/2YI1FRNoBV7cyNeVkywOUlJw8KLWkZaq6Wt160iivh5s333yTu+66ixdeeIGRI0cyf/58xo8fT2ZmJt26dTuh/JdffsnUqVOZN28el156KYsWLWLixIls3ryZ008/3QufQEREPBqk6mosVB044AhWDS0vAE0PHD++XHNCVwddrqC98vo6NyNHjuTMM8/k2WefBcButxMXF8cdd9zBfffdd0L5KVOmUFJSwvvvv+88dtZZZzFkyBBeeOGFk76f1rkRERG3aM7ilq4KUy1t5Tqep1u9bDb4/vvOsc5NZWUlmzZtYu7cuc5jFouFcePGsW7dugZfs27dOu666656x8aPH8/SpUsbLF9RUUFFRYXzeVFR0alXXERE5Hjear1qLVcsj1BXYCBEREBYGHTtCtHREBcHffvCwIEevTdeDTf5+fnU1NQQHR1d73h0dDQZGRkNviY3N7fB8rm5uQ2WnzdvHo888ohrKiwiItJRuHMwu5e1dv2hdmPu3LkUFhY6H9nZ2d6ukoiIiLiRV1tuIiMjsVqt5OXl1Tuel5dHTExMg6+JiYlpUXmbzYbNZnNNhUVERKTN82rLjZ+fH8OGDWPVqlXOY3a7nVWrVpHayDLrqamp9coDpKWlNVpeREREOhevTwW/6667mD59OsOHD2fEiBHMnz+fkpISZs6cCcC1115Ljx49mDdvHgB33nkn5513Hk8//TSXXHIJixcvZuPGjbz44ove/BgiIiLSRng93EyZMoWDBw/y4IMPkpuby5AhQ/joo4+cg4azsrKwWH5uYBo1ahSLFi3ij3/8I3/4wx/o168fS5cu1Ro3IiIiArSBdW48TevciIiItD8t+f7u8LOlREREpHNRuBEREZEOReFGREREOhSvDyj2tNohRtqGQUREpP2o/d5uzlDhThduiouLAYiLi/NyTURERKSliouLCQsLa7JMp5stZbfb2b9/PyEhIRiG4dJrFxUVERcXR3Z2tmZiuZHus+foXnuG7rNn6D57jjvutWmaFBcX071793pLxDSk07XcWCwWevbs6db3CA0N1X84HqD77Dm6156h++wZus+e4+p7fbIWm1oaUCwiIiIdisKNiIiIdCgKNy5ks9l46KGHtAu5m+k+e47utWfoPnuG7rPnePted7oBxSIiItKxqeVGREREOhSFGxEREelQFG5ERESkQ1G4ERERkQ5F4cZFnnvuORISEvD392fkyJF8/fXX3q5Su/PZZ59x2WWX0b17dwzDYOnSpfXOm6bJgw8+SGxsLAEBAYwbN46dO3fWK3P48GGmTZtGaGgo4eHhXH/99Rw9etSDn6JtmzdvHmeeeSYhISF069aNiRMnkpmZWa9MeXk5s2bNomvXrgQHBzN58mTy8vLqlcnKyuKSSy4hMDCQbt268bvf/Y7q6mpPfpQ27/nnn2fQoEHORcxSU1P58MMPned1n93jiSeewDAM5syZ4zyme+0aDz/8MIZh1HskJSU5z7ep+2zKKVu8eLHp5+dnvvLKK+b27dvNG2+80QwPDzfz8vK8XbV2Zfny5eb9999vvvvuuyZgLlmypN75J554wgwLCzOXLl1qfvPNN+Yvf/lLs3fv3mZZWZmzzEUXXWQOHjzY/Oqrr8zPP//c7Nu3rzl16lQPf5K2a/z48eaCBQvM7777zty6dat58cUXm/Hx8ebRo0edZW655RYzLi7OXLVqlblx40bzrLPOMkeNGuU8X11dbZ5++unmuHHjzC1btpjLly83IyMjzblz53rjI7VZy5YtMz/44APz+++/NzMzM80//OEPpq+vr/ndd9+Zpqn77A5ff/21mZCQYA4aNMi88847ncd1r13joYceMgcMGGDm5OQ4HwcPHnSeb0v3WeHGBUaMGGHOmjXL+bympsbs3r27OW/ePC/Wqn07PtzY7XYzJibGfOqpp5zHCgoKTJvNZr7xxhumaZrmjh07TMDcsGGDs8yHH35oGoZh/vTTTx6re3ty4MABEzA//fRT0zQd99TX19d8++23nWXS09NNwFy3bp1pmo4QarFYzNzcXGeZ559/3gwNDTUrKio8+wHamYiICPPll1/WfXaD4uJis1+/fmZaWpp53nnnOcON7rXrPPTQQ+bgwYMbPNfW7rO6pU5RZWUlmzZtYty4cc5jFouFcePGsW7dOi/WrGPZvXs3ubm59e5zWFgYI0eOdN7ndevWER4ezvDhw51lxo0bh8ViYf369R6vc3tQWFgIQJcuXQDYtGkTVVVV9e5zUlIS8fHx9e7zwIEDiY6OdpYZP348RUVFbN++3YO1bz9qampYvHgxJSUlpKam6j67waxZs7jkkkvq3VPQ77Sr7dy5k+7du9OnTx+mTZtGVlYW0Pbuc6fbONPV8vPzqampqffDAoiOjiYjI8NLtep4cnNzARq8z7XncnNz6datW73zPj4+dOnSxVlGfma325kzZw5nn302p59+OuC4h35+foSHh9cre/x9bujnUHtOfrZt2zZSU1MpLy8nODiYJUuWkJKSwtatW3WfXWjx4sVs3ryZDRs2nHBOv9OuM3LkSBYuXEhiYiI5OTk88sgjnHvuuXz33Xdt7j4r3Ih0UrNmzeK7775j7dq13q5Kh5WYmMjWrVspLCzknXfeYfr06Xz66aferlaHkp2dzZ133klaWhr+/v7erk6HNmHCBOffBw0axMiRI+nVqxdvvfUWAQEBXqzZidQtdYoiIyOxWq0njAjPy8sjJibGS7XqeGrvZVP3OSYmhgMHDtQ7X11dzeHDh/WzOM7tt9/O+++/z5o1a+jZs6fzeExMDJWVlRQUFNQrf/x9bujnUHtOfubn50ffvn0ZNmwY8+bNY/DgwfzjH//QfXahTZs2ceDAAYYOHYqPjw8+Pj58+umnPPPMM/j4+BAdHa177Sbh4eH079+fXbt2tbnfaYWbU+Tn58ewYcNYtWqV85jdbmfVqlWkpqZ6sWYdS+/evYmJial3n4uKili/fr3zPqemplJQUMCmTZucZVavXo3dbmfkyJEer3NbZJomt99+O0uWLGH16tX07t273vlhw4bh6+tb7z5nZmaSlZVV7z5v27atXpBMS0sjNDSUlJQUz3yQdsput1NRUaH77EJjx45l27ZtbN261fkYPnw406ZNc/5d99o9jh49yg8//EBsbGzb+5126fDkTmrx4sWmzWYzFy5caO7YscO86aabzPDw8HojwuXkiouLzS1btphbtmwxAfNvf/ubuWXLFnPv3r2maTqmgoeHh5vvvfee+e2335qXX355g1PBzzjjDHP9+vXm2rVrzX79+mkqeB233nqrGRYWZn7yySf1pnOWlpY6y9xyyy1mfHy8uXr1anPjxo1mamqqmZqa6jxfO53zwgsvNLdu3Wp+9NFHZlRUlKbNHue+++4zP/30U3P37t3mt99+a953332mYRjmypUrTdPUfXanurOlTFP32lXuvvtu85NPPjF3795tfvHFF+a4cePMyMhI88CBA6Zptq37rHDjIv/85z/N+Ph408/PzxwxYoT51VdfebtK7c6aNWtM4ITH9OnTTdN0TAd/4IEHzOjoaNNms5ljx441MzMz613j0KFD5tSpU83g4GAzNDTUnDlzpllcXOyFT9M2NXR/AXPBggXOMmVlZeZtt91mRkREmIGBgeakSZPMnJycetfZs2ePOWHCBDMgIMCMjIw07777brOqqsrDn6Ztu+6668xevXqZfn5+ZlRUlDl27FhnsDFN3Wd3Oj7c6F67xpQpU8zY2FjTz8/P7NGjhzllyhRz165dzvNt6T4bpmmarm0LEhEREfEejbkRERGRDkXhRkRERDoUhRsRERHpUBRuREREpENRuBEREZEOReFGREREOhSFGxEREelQFG5EpFMyDIOlS5d6uxoi4gYKNyLicTNmzMAwjBMeF110kberJiIdgI+3KyAindNFF13EggUL6h2z2Wxeqo2IdCRquRERr7DZbMTExNR7REREAI4uo+eff54JEyYQEBBAnz59eOedd+q9ftu2bfziF78gICCArl27ctNNN3H06NF6ZV555RUGDBiAzWYjNjaW22+/vd75/Px8Jk2aRGBgIP369WPZsmXOc0eOHGHatGlERUUREBBAv379TghjItI2KdyISJv0wAMPMHnyZL755humTZvGr3/9a9LT0wEoKSlh/PjxREREsGHDBt5++20+/vjjeuHl+eefZ9asWdx0001s27aNZcuW0bdv33rv8cgjj3DVVVfx7bffcvHFFzNt2jQOHz7sfP8dO3bw4Ycfkp6ezvPPP09kZKTnboCItJ7Lt+IUETmJ6dOnm1ar1QwKCqr3+POf/2yapmP38ltuuaXea0aOHGneeuutpmma5osvvmhGRESYR48edZ7/4IMPTIvFYubm5pqmaZrdu3c377///kbrAJh//OMfnc+PHj1qAuaHH35omqZpXnbZZebMmTNd84FFxKM05kZEvOL888/n+eefr3esS5cuzr+npqbWO5eamsrWrVsBSE9PZ/DgwQQFBTnPn3322djtdjIzMzEMg/379zN27Ngm6zBo0CDn34OCgggNDeXAgQMA3HrrrUyePJnNmzdz4YUXMnHiREaNGtWqzyoinqVwIyJeERQUdEI3kasEBAQ0q5yvr2+954ZhYLfbAZgwYQJ79+5l+fLlpKWlMXbsWGbNmsVf//pXl9dXRFxLY25EpE366quvTnienJwMQHJyMt988w0lJSXO81988QUWi4XExERCQkJISEhg1apVp1SHqKgopk+fzuuvv878+fN58cUXT+l6IuIZarkREa+oqKggNze33jEfHx/noN23336b4cOHc8455/Df//6Xr7/+mn//+98ATJs2jYceeojp06fz8MMPc/DgQe644w6uueYaoqOjAXj44Ye55ZZb6NatGxMmTKC4uJgvvviCO+64o1n1e/DBBxk2bBgDBgygoqKC999/3xmuRKRtU7gREa/46KOPiI2NrXcsMTGRjIwMwDGTafHixdx2223ExsbyxhtvkJKSAkBgYCArVqzgzjvv5MwzzyQwMJDJkyfzt7/9zXmt6dOnU15ezt///nfuueceIiMj+dWvftXs+vn5+TF37lz27NlDQEAA5557LosXL3bBJxcRdzNM0zS9XQkRkboMw2DJkiVMnDjR21URkXZIY25ERESkQ1G4ERERkQ5FY25EpM1Rb7mInAq13IiIiEiHonAjIiIiHYrCjYiIiHQoCjciIiLSoSjciIiISIeicCMiIiIdisKNiIiIdCgKNyIiItKhKNyIiIhIh/L/AfiXuQsj2e+IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## plot loss cruve\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "idx_list = list(range(len(valid_list)))\n",
    "\n",
    "#fig = plt.figure()\n",
    "plt.plot(idx_list,train_list,'s-',color = 'r',label=\"Train loss\")  # train curve\n",
    "plt.plot(idx_list,valid_list,'o-',color = 'g',label=\"Valid loss\", alpha = 0.5)  # validation curve\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend() \n",
    "plt.title(\"Train-Valid Loss Curve\")\n",
    "plt.savefig(\"Q1_loss_curve\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b5acd78",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}