L08_logistic/code/logistic-regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STAT 479: Deep Learning (Spring 2019)  \n",
    "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n",
    "Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat479-ss2019/  \n",
    "GitHub repository: https://github.com/rasbt/stat479-deep-learning-ss19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.1\n",
      "IPython 7.2.0\n",
      "\n",
      "torch 1.0.1\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Runs on CPU or GPU (if available)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of *classic* logistic regression for binary class labels."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing a toy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAACqCAYAAAD1E6s4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGWRJREFUeJzt3X+QXWV9x/H3l7DqUikrzVpkkyVYKVYgFd2htnFaBeSX/JIqVWuH0Y4ZpuOoGUSgzGhEO4bJVKq1rROVsa1RE2v4pTCARMapM7FuIAnESAcsP7LQEguJP7KWDXz7x703uXv3nHPPuefce55z7uc1s5Pdu3fPffbmPPs9z/f5Ps8xd0dERKQqDiu7ASIiIlkocImISKUocImISKUocImISKUocImISKUocImISKUocImISKUocImISKUocImISKUcXsaLLl682JctW1bGS4sUYuvWrT9z9/Gy29GiPiV1kLZflRK4li1bxvT0dBkvLVIIM3us7Da0U5+SOkjbr5QqFBGRSlHgEgmUmS0ys/vN7Ntlt0UkJKWkCqVabr5/hrV3PsSTe2c5dmyUK88+kYtPncj9XOnqQ8Au4DfLbohISDTikkQ33z/DNZseYGbvLA7M7J3lmk0PcPP9M7meK8nMbAnwVuBLZbdFJDQKXJJo7Z0PMTv3/LzHZueeZ+2dD+V6rnT1d8BHgRfinmBmK81s2sym9+zZM7iWiZRMgUsSPbl3NvXjWZ4r8czsfOBpd9+a9Dx3X+fuU+4+NT4eTGW+SN8pcEmiY8dGUz+e5bmSaAVwoZk9CnwDON3Mvlpuk0TCocAlia48+0RGRxbNe2x0ZBFXnn1irudKPHe/xt2XuPsy4J3AZnd/T8nNEgmGqgolUasiMK5SsLOK8E9fP8H3frIn2KpCVT2KVJ8Cl3R18akTkX/cW1WErYKMmb2zfGvrDJ++5JQgg0FUe6/Z9ABAkO0FcPd7gXtLboZIUJQqlJ5VrYowrr1XbNzO8Vd/hxVrNqt0X6QCNOKSnhVVRTio9F1cu553B6oxAhMRjbgkhyKqCAe5aDlNu0IeMYpIgwKX9KyIKsJBphuj2htF685EwqZUoWTWntobO2KEFx9+GPtm53pK8w1y0XJnheRhZgfThO207kwkbApckklnZd6z++cYHVnEDX/22p7mhY4dG2UmIkj1K3i0V0h2/i6gdWciVaBUocS6+f4ZVqzZPK/irujUXpmLli8+dYJPX3IKE2OjGDDRXIe29s6HVGUoEjCNuCRS3JqnzqDV0mtqL2mBc9HVhnHHixuBqcpQJEwKXBIpbmS1KOO8UJrgE7XAOSqIfHjDNj5x204+fsFJmQNJmqCUNJpU4BIJh1KFEilpzVPa1F6eUveoIAKNObVeyuXTpDi1u71INShwSaS4EdQis4MjL2jMC336klMACp0PSwoWvcyppQlK2t1epBoKCVxmdqOZPW1mDxZxPClf3JqnVpqwNfJqjbSiRlZR1YJwKFhEFX+0dAsWWUdBaYKSdrcXqYaiRlxfAc4p6FgSgM6Ku9YIq11r5JM0Hxbl2LHRrmnEbouFs46C0gSlqCrDUDcMFhlmhRRnuPv3zWxZEceScLQXTRx/9Xcin5M08mmNyqLWSXUrhGi97upbd7J3dm7e83oZBXW7PUv78xSoRMKmqkJJpdtC4ajvTTSDQ1SwWLVhW+TrtAfCVgD5xG07eXZ/I3iNjY6w+sLsVYWt4ykoiVTfwAKXma0EVgJMTk4O6mWlIFeefWLiLhNx34sLFml2zIja2eJXzx1g9a07WbVhG8eOjfLmV48HfeNKESnewAKXu68D1gFMTU0tXAgkwcl6d+Msi4W7BcLW8TrTiXPP+8HU4czeWb665fGD39OCYZHhoFShRMp6d+Osabg0c069rJ/SgmGR+iskcJnZ14E3AYvNbDfwcXf/chHHlnIMYheJbsEuLp3YjRYMi9RbUVWF7yriOBKOuD/+M3tnWbFm87zRUWdKsah5p6h0YhpaMCxSb0oVSqSk0U77XBKwIKVY1LxTZzpx7IgRfvnrA8y9ED9FWocFw2a2FPgX4BjgBWCdu3+23FaJhEOBSyJ1G+20b7vUbUSUJ8XYmU7s1+guMAeAK9z9PjM7EthqZne7+4/LbphICBS4JFL7aKfb1k1pZHlultuZTB13NJ+6+JTUx64Cd38KeKr5+S/MbBcwAShwiaDAJQlao50VazbHrrna/9yBg4uDk6Sdd4qqZly1YRvTjz3D1HFHD939spo70pwK/DDie1obKUNJu8NLV0n7/EXcmmuBLPNOUdWMDqzf8jirb91Z6N2XQ2dmLwW+BXzY3X/e+X13X+fuU+4+NT4+PvgGipREgUu6Stp8dt9s/Girl41q41KKDgv2LOz2M1VmZiM0gtZ6d99UdntEQqJUocRqzTXN7J09eOfjiY45p7jqw4mxUX5w9emZX7OXtVt1K383MwO+DOxy98+U3R7pTftc7dgRI7jDvtm5OhURlUYjLonUftsROHQfrjS3H8lTkn7l2ScSfTMUeNkRI8Nyv6wVwF8Ap5vZtubHeWU3StLrvG3Ps/vn2Ds7d/AWPqs2bGNZxH3oJB2NuOpqx0a45zrYtxuOWgJnfAyWX5r6x6Pmmlqibj+SZZ/CJBefOsH0Y8+wfsvjtE+fjY4s4uMXnFToa4XK3f8dYuO3VEBS/wEOntvDUGDUDwpcdbRjI9z2QZhrptz2PdH4GlIHr27zRp23Hymy033q4lOYOu7oBQEK6h+0pB6yzLtqf83sFLjq6J7rDgWtlrnZxuMpA1e3uaZ+zytFLTwetlJ4qa6sc7V1LDDqJ81x1dG+3dkejxA1d9WSZl7p5vtnWLFmM8cXlMdP2vRXJDRJ/SdK3QqM+k0jrjo6akkjPRj1eEqdO2fEVRVG6cfoKO6KVFeqEqKofTbdG0s6DBbM39awwKivFLjq6IyPzZ/jAhgZbTyeQa9zV/24JUqaOyaLlC3NdmVZtjSTaPUPXDmr6yqp9fuV9Hv3Y3SU5o7JImVKm2kouphpGNU7cBVQXVdZyy8t7Xfsx+goquz+za8eZ+2dD7FqwzZduUqpbr5/his2bj+43rFldu55rti4XedoweoduAqorpPseh0ddUuhtF+pqspQQtE6FzuDVkvn4n3QOZpXvasKC6iu66sdG+GGk2H1WOPfHRvLblEhkvY2jKs27NxpoHOHjk6qMpRQdFts3E7naDHqPeIqoLqub2qexozK4yeNkrIWdKjKUEKR9ZxLer4KN9Kp94jrjI81quna9VBd1xdJacyaSgpOWQNR3HyZqgxl0LKec3HPz5p1GGb1DlzLL4ULPgdHLQWs8e8FnwtjRBN6GrMAnWnBpDspZw1ERW/uK9KruHPxPW+YzHSOKv2dXr1ThVBqdV2ikNOYBYhKC3YuvGxppUSyFHQUvbmvSK+SzsWoPTfjzlGlv9Orf+AKVUGLhActbQ4+7k7GcbsG9BKItB5GQhF3LsbN9Uad51pkn54CV1lKXiQcqcti7Swl6El3Mp4YG40MTlEb665Ys1kjKqmkqAAFxPYhLbJPr5DAZWbnAJ8FFgFfcvc1RRy39kJKY6aocsxS+Zf3zshapyVVFnX+rtqwjdGRw5ide2Hec1t9qNUvlP7uLndxhpktAv4BOBd4DfAuM3tN3uPKgKWocsySg89bPKGJaqmMiPWYcany/R1Bq2Vm7ywr1mwG4AdXn85/rXkrP7j6dAWtGEVUFZ4GPOzuP3X354BvABcVcFwZpBRVjlkq/5IWIaehiWqphFamYt8TgB/MVEz9/O7Mh1L5e3pFpAongPbyuN3AH3Q+ycxWAisBJicnC3hZKVSKKsdeKv+0G7zUWkym4poXfZNbfv3GzIdLWnSvxcmHFDHisojHFlQ9u/s6d59y96nx8fECXjZwVdvOKcVi7byjqCy0TksqISZT8dv8LPIPI8DY6AgTCRdgUVkFLU6er4gR125gadvXS4AnCzhudYW8nVNc5WDKKsdBlaAP+zotFTyVoJdbIMVkKuyoJfz5aydZv+XxBcs/Vl94EhefOhG7KD8qq9CPe9xVWRGB60fACWZ2PDADvBN4dwHHra5Qd6XvFlBDqnJkeNdptRU8vYXGheGPzOxWd/9xuS2rsV4vNhPWY35q+SmJC5CzpN415ztf7sDl7gfM7APAnTSuDm909525W1ZloW7nFGpAlU4HC54AzKxV8KTA1S+99o0umYqki68sWQXN+c5XyDoud78duL2IYwUjz52TQ93OqY8BVRPHhVLB06Dl6Rs5MhVpswpanDxfvTfZ7VVMiWvqAos8u9L3s6gjLnDmDKiaOC6cCp4GrU99oyiDLIyqAm35FCVvSq3X7Zz6XdTRp/0RNXFcOBU8DVoF9g4d1jnfKApcnXZsjE7zQbaUWi/pg37PQfVpf0RNHBdOBU959JLmD3HvUImlwNWuNeKJ0++0wSCKOvpQOaiJ42Kp4CmHPFmLwKpqJZ7muNpFjXhaBpE2CDzPHkeLhYvn7re7+++6+++4+9+U3Z7KGMI7iw8jBa52SSObQdw5OU9RR4k0cSzB6CVrUWRBVNV2zKkopQrbxZaxLx1MCiEqz37CWY2vN60MOu+uiWMJQtalKFGpxU3vhzuugnOvz9bXQt4xp2bCG3GVecUSwohn+aWw6kFYvbfxutu/1ntZvsiwydqH46YHZp/J3teUphyYsAJX3vVTeS2/tJESPGopYI1/B5EijKOOIJJN1j6clEJs9bW0F9Oh7phTQ2GlCkPYkiikyiJ1BJHssvThuNRiS+viOU36L9Qdc2oorBGX/lDPV9EqQ5HKiEottrNF6bMeIUw1DImwAlc//1BXsdpHHUGkuzx9u5VaHD164fdGRsGfX/g4zL+Ybr3+ppVw+GjzWAFMNdRYWKnCfm27UlS1T56Nd7sdb/Rljcdmn114bK3mF4mWpW8n9bdzr2983dnX7rkuOf3X+fqzzzT+Zl2yTv20j8IKXP36Q13E3FnRpa5RJ3zLvicaV2+Pb4HzP3Pod9i3+1CKQp1CJH3f7tbfbvtgY3S06sGFr5F0MR3CvPwQCitwQX+KI4qYOyv6BE3apQMAh+kbG59u/5rWhohESdO3d2yEmy6PT/tBfF/udjGteflShBe4+qGIap+iT9BUP+ew9SsLO5yu6EQauvXt1kgrKWi1xPXJpItpVRKWIqzijH4posih6MKRtD+XZnJYZFh169tdMxtteunLKqAqxXAEriIWFvdygiZVO3Urw22xRdGP64pOpHvfTnuB12uwCW3TgiExHKlCyD93lrVwpFsxR+fxRo6AuV/NP8bIKPz+u+fPcbUeb+9kRVc7ilRJL6k8OwxeMhZdxVvk60tfDE/gyitrcIgr5rjjqkM/13nCx73G5BviX1sbe4rEi1tio1FRpSlwpdFLcIhLUcw+0zhe1M/FXbklXdGpHFckntZC1pICVxq9BIekPdCKDCoqxxVJplRe7QxHcUZevQSHpIneIoOK9jMUkSGTK3CZ2TvMbKeZvWBmU0U1Kji9BIfll0bvf9bt57KKqk48bASe+1W19mUUEUkp74jrQeAS4PsFtCVcva7VOPf67j+Xd/PfznLc0aPBrLmlTcc9zaq40bCISIdcc1zuvgvAzIppTah6neDt9nNFVQS25/BvOHn+PmxwqJrxwKyqDwNnZmuBC4DngEeA97r73nJbVXFaLlI75u75D2J2L/ARd59OeM5KYCXA5OTk6x977LHcr1t5N5wcs13M0ujNPtNYPQZk+D/N81pDzMy2unvh6XEzOwvY7O4HzOx6AHe/qtvPTU1N+fR0bPcbXp0Xh6By+ICl7VddU4Vm9l0zezDi46IsDXL3de4+5e5T4+PjWX40bHnSb/2oCMw6f6bqw6C4+13ufqD55RZAVTZ5JFUES2V1DVzufqa7nxzxccsgGhi01tXcvidYMJ+URmyQ8d7noOLm4wZRKCJFex9wR9w3zWylmU2b2fSePXsG2KwK0XKRWlI5fB5xV3Ob3p8u8CTtV5g1CLbE7Z2WplBEBiJNFsPMrgUOAOvjjlPbLEaRtFyklnIVZ5jZ24C/B8aB75jZNnc/u5CWVUHSVVua4od5xRsRc1297oDRbacNTVKXyt3PTPq+mV0GnA+c4UVMQg+zft1VXUqVt6rwJuCmgtpSPUm7Y0C6wNMKMnFFFUWmNLSDQPDM7BzgKuBP3H1/2e2pPG35VEva8imPqKu5TmkDj25IJw2fB14M3N1cZrLF3S8vt0kVpwu22lHgyqNbqg/SBx6lNARw91eV3QaR0Kk4I6/llzbWQV3yxXzFD3FFFaDdLkRE2mjEVZQiculR9+fKsrOGdggQkSGgwFWkonPpWW6nohtKisiQUKowZHHzZlEFH9ohQESGhAJXqHZsBGI2L44q+NAOASIyJBS4QnXPdURvlmvRBR/aIUBEhoQCV6hiR0oePWfV6z3D0tK9vEQkEApcoYodQS2NfjyunL6Iwoy8mwmLiBRIVYWh6mVBcr92CMhS3Sgi0mcacYWqnyOorFT4ISIB0YirSEUvAA5ljzXtoygiAdGIqyihzwPlKa7od+GHiEgGClxFCXkBcN6gGlLaUkSGnlKFRQl5HqiI4opQ0pYiMvQ04ipKyAuAQw6qIiIZKXAlyTIvFPI8UMhBVUQkIwWuOFnnhUKeBxpkUNUOGyLSZ5rjitPLvFA/54HylNoXca+wtG3UrVVEpM8UuOKENC9UREAYRHGFdtgQkQFQqjBOSPNCIZfatwsp2ItIbSlwxQmp2KIqASGkYF9xZvYRM3MzW1x2W0RCo8AVJ6Rii6oEhJCCfYWZ2VLgLcDjZbdFJES55rjMbC1wAfAc8AjwXnffW0TDghDKottedoovw6CKQOrvBuCjwC1lN0QkRHmLM+4GrnH3A2Z2PXANcFX+Zsk8VQoIoQT7ijKzC4EZd99uZt2euxJYCTA5OTmA1omEIVfgcve72r7cArw9X3MklgJCbZjZd4FjIr51LfDXwFlpjuPu64B1AFNTU15YA0UCV2Q5/PuADXHf1NWhSIO7nxn1uJmdAhwPtEZbS4D7zOw0d//vATZRJGhdA1fS1aG739J8zrXAAWB93HFqcXVY9P22RNq4+wPAy1tfm9mjwJS7/6y0RokEqGvgirs6bDGzy4DzgTPcvZoBKQ3tCiEiEoRc5fBmdg6NYowL3X1/MU0KVFUWAUttuPsyjbZEFsq7juvzwJHA3Wa2zcy+UECbwlSVRcAiIjWXt6rwVUU1JHhHLWnuFB/xuIiIDIx2zkhLu0KIiARBgSutkLaAEhEZYrqtSRZaBCwiUjqNuEREpFIUuEREpFIUuEREpFKsjM0uzGwP8NgAXmoxENoCTrUpndDbdJy7j5fZmHY99qkQ3+MoVWhnFdoI4bczVb8qJXANiplNu/tU2e1opzalozb1X1V+nyq0swpthOq0sxulCkVEpFIUuEREpFLqHrjWld2ACGpTOmpT/1Xl96lCO6vQRqhOOxPVeo5LRETqp+4jLhERqRkFLhERqZTaBy4zW2tmPzGzHWZ2k5mNBdCmd5jZTjN7wcxKLU01s3PM7CEze9jMri6zLc323GhmT5vZg2W3pcXMlprZ98xsV/P/7UNlt6kXZrbazGaa987bZmbnxTyvtHMibX81s0fN7IHm7zE9wPYlvjdm9mIz29D8/g/NbNmg2tbWhq7nq5m9ycz2tZ0L1brNhbvX+gM4Czi8+fn1wPUBtOn3gBOBe4GpEtuxCHgEeCXwImA78JqS35s/Bl4HPFj2/1Nbm14BvK75+ZHAf5b9PvX4e6wGPhLyOZG2vwKPAosH/P51fW+AvwK+0Pz8ncCGEv6fu56vwJuAbw+6bUV91H7E5e53ufuB5pdbgNLv/Ojuu9z9obLbAZwGPOzuP3X354BvABeV2SB3/z7wTJlt6OTuT7n7fc3PfwHsAibKbVXflHpOhNhf26R5by4C/rn5+b8BZ5iZDbCNQ3G+1j5wdXgfcEfZjQjIBNB+W+fd1OwEL1oz9XMq8MNyW9KzDzTTcDea2csivh/SOZHUXx24y8y2mtnKAbUnzXtz8DnNALwP+K2BtC5Cl/P1D81su5ndYWYnDbRhOdXiflxm9l3gmIhvXevutzSfcy1wAFgfSpsCEHUlqPURMczspcC3gA+7+8/Lbk+UpPMO+CfgkzT+jz8J/C2N4DDvEBE/W+g5UVB/XeHuT5rZy4G7zewnzdF6P6V5b4LpU13O1/to7Av4y+Zc583ACYNuY69qEbjc/cyk75vZZcD5wBneTPCW3aZA7AaWtn29BHiypLYEzcxGaPwRWO/um8puT5y0552ZfRH4dsS3+n5OFNFf3f3J5r9Pm9lNNNJ4/Q5cad6b1nN2m9nhwFGUkPrudr62BzJ3v93M/tHMFrt7yBvwHlT7VKGZnQNcBVzo7vvLbk9gfgScYGbHm9mLaEwm31pym4LTnKP4MrDL3T9Tdnt6ZWavaPvybUBU5Wap50Sa/mpmv2FmR7Y+p1HQMYgq1DTvza3AZc3P3w5sHtTFckua89XMjmnNvZnZaTRiwf8OrpU5lV0d0u8P4GEaOedtzY8vBNCmt9G4Mvs/4H+AO0tsy3k0qo4eoZGqKfu9+TrwFDDXfI/+MoA2vZFGumdH23l0Xtnt6uH3+FfggebvcSvwiubjxwK3h3BOxPXX9jbSqOrb3vzYOcg2Rr03wHU0Ai3AS4BvNn+P/wBeWcL/c+T5ClwOXN58zgea7912GkUwf1T2+ZnlQ1s+iYhIpdQ+VSgiIvWiwCUiIpWiwCUiIpWiwCUiIpWiwCUiIpWiwCUiIpWiwCUiIpXy/xTkZk9heOeAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x180 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "##########################\n",
    "### DATASET\n",
    "##########################\n",
    "\n",
    "data = np.genfromtxt('data/toydata.txt', delimiter='\\t')\n",
    "x = data[:, :2].astype(np.float32)\n",
    "y = data[:, 2].astype(np.int64)\n",
    "\n",
    "np.random.seed(123)\n",
    "idx = np.arange(y.shape[0])\n",
    "np.random.shuffle(idx)\n",
    "X_test, y_test = x[idx[:25]], y[idx[:25]]\n",
    "X_train, y_train = x[idx[25:]], y[idx[25:]]\n",
    "mu, std = np.mean(X_train, axis=0), np.std(X_train, axis=0)\n",
    "X_train, X_test = (X_train - mu) / std, (X_test - mu) / std\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(7, 2.5))\n",
    "ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1])\n",
    "ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1])\n",
    "ax[1].scatter(X_test[y_test == 1, 0], X_test[y_test == 1, 1])\n",
    "ax[1].scatter(X_test[y_test == 0, 0], X_test[y_test == 0, 1])\n",
    "plt.xlim([x[:, 0].min()-0.5, x[:, 0].max()+0.5])\n",
    "plt.ylim([x[:, 1].min()-0.5, x[:, 1].max()+0.5])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Low-level implementation with manual gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "def custom_where(cond, x_1, x_2):\n",
    "    return (cond * x_1) + ((1-cond) * x_2)\n",
    "\n",
    "\n",
    "class LogisticRegression1():\n",
    "    def __init__(self, num_features):\n",
    "        self.num_features = num_features\n",
    "        self.weights = torch.zeros(1, num_features, \n",
    "                                   dtype=torch.float32, device=device)\n",
    "        self.bias = torch.zeros(1, dtype=torch.float32, device=device)\n",
    "\n",
    "    def forward(self, x):\n",
    "        linear = torch.add(torch.mm(x, self.weights.t()), self.bias).view(-1)\n",
    "        probas = self._sigmoid(linear)\n",
    "        return probas\n",
    "        \n",
    "    def backward(self, x, y, probas):  \n",
    "        \n",
    "        grad_loss_out = y - probas.view(-1)\n",
    "        grad_loss_w = -torch.mm(x.t(), grad_loss_out.view(-1, 1)).t()\n",
    "        grad_loss_b = -torch.sum(grad_loss_out)\n",
    "        return grad_loss_w, grad_loss_b\n",
    "            \n",
    "    def predict_labels(self, x):\n",
    "        probas = self.forward(x)\n",
    "        labels = custom_where(probas >= .5, 1, 0)\n",
    "        return labels    \n",
    "            \n",
    "    def evaluate(self, x, y):\n",
    "        labels = self.predict_labels(x).float()\n",
    "        accuracy = torch.sum(labels.view(-1) == y.float()).item() / y.size(0)\n",
    "        return accuracy\n",
    "    \n",
    "    def _sigmoid(self, z):\n",
    "        return 1. / (1. + torch.exp(-z))\n",
    "    \n",
    "    def _logit_cost(self, y, proba):\n",
    "        tmp1 = torch.mm(-y.view(1, -1), torch.log(proba.view(-1, 1)))\n",
    "        tmp2 = torch.mm((1 - y).view(1, -1), torch.log(1 - proba.view(-1, 1)))\n",
    "        return tmp1 - tmp2\n",
    "    \n",
    "    def train(self, x, y, num_epochs, learning_rate=0.01):\n",
    "        epoch_cost = []\n",
    "        for e in range(num_epochs):\n",
    "            \n",
    "            #### Compute outputs ####\n",
    "            probas = self.forward(x)\n",
    "            \n",
    "            #### Compute gradients ####\n",
    "            grad_w, grad_b = self.backward(x, y, probas)\n",
    "\n",
    "            #### Update weights ####\n",
    "            self.weights -= learning_rate * grad_w\n",
    "            self.bias -= learning_rate * grad_b\n",
    "            \n",
    "            #### Logging ####\n",
    "            cost = self._logit_cost(y, self.forward(x)) / x.size(0)\n",
    "            print('Epoch: %03d' % (e+1), end=\"\")\n",
    "            print(' | Train ACC: %.3f' % self.evaluate(x, y), end=\"\")\n",
    "            print(' | Cost: %.3f' % cost)\n",
    "            epoch_cost.append(cost)\n",
    "        return epoch_cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | Train ACC: 0.973 | Cost: 0.055\n",
      "Epoch: 002 | Train ACC: 0.973 | Cost: 0.053\n",
      "Epoch: 003 | Train ACC: 0.973 | Cost: 0.051\n",
      "Epoch: 004 | Train ACC: 0.973 | Cost: 0.049\n",
      "Epoch: 005 | Train ACC: 0.973 | Cost: 0.048\n",
      "Epoch: 006 | Train ACC: 0.973 | Cost: 0.047\n",
      "Epoch: 007 | Train ACC: 0.973 | Cost: 0.046\n",
      "Epoch: 008 | Train ACC: 0.973 | Cost: 0.045\n",
      "Epoch: 009 | Train ACC: 0.973 | Cost: 0.044\n",
      "Epoch: 010 | Train ACC: 0.987 | Cost: 0.043\n",
      "Epoch: 011 | Train ACC: 0.987 | Cost: 0.042\n",
      "Epoch: 012 | Train ACC: 0.987 | Cost: 0.041\n",
      "Epoch: 013 | Train ACC: 0.987 | Cost: 0.041\n",
      "Epoch: 014 | Train ACC: 0.987 | Cost: 0.040\n",
      "Epoch: 015 | Train ACC: 0.987 | Cost: 0.039\n",
      "Epoch: 016 | Train ACC: 0.987 | Cost: 0.039\n",
      "Epoch: 017 | Train ACC: 1.000 | Cost: 0.038\n",
      "Epoch: 018 | Train ACC: 1.000 | Cost: 0.038\n",
      "Epoch: 019 | Train ACC: 1.000 | Cost: 0.037\n",
      "Epoch: 020 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 021 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 022 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 023 | Train ACC: 1.000 | Cost: 0.035\n",
      "Epoch: 024 | Train ACC: 1.000 | Cost: 0.035\n",
      "Epoch: 025 | Train ACC: 1.000 | Cost: 0.034\n",
      "Epoch: 026 | Train ACC: 1.000 | Cost: 0.034\n",
      "Epoch: 027 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 028 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 029 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 030 | Train ACC: 1.000 | Cost: 0.032\n",
      "\n",
      "Model parameters:\n",
      "  Weights: tensor([[5.0453, 3.4349]])\n",
      "  Bias: tensor([-0.7931])\n"
     ]
    }
   ],
   "source": [
    "X_train_tensor = torch.tensor(X_train, dtype=torch.float32, device=device)\n",
    "y_train_tensor = torch.tensor(y_train, dtype=torch.float32, device=device)\n",
    "\n",
    "model1 = LogisticRegression1(num_features=2)\n",
    "epoch_cost = model1.train(X_train_tensor, y_train_tensor, num_epochs=30, learning_rate=0.1)\n",
    "\n",
    "print('\\nModel parameters:')\n",
    "print('  Weights: %s' % model1.weights)\n",
    "print('  Bias: %s' % model1.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluating the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VPW9x/H3NwtJCBC2hC2EfZEdiaCAVAUX1IKtVFyqtti6VFtta29te7vord3udalVa7HihSp1x1IV0YKKKLITdjTse1gEwhIgyff+kUM7N01gYDKZTPJ5Pc88mTlz5vA9z0A+/JbzO+buiIiInKmEWBcgIiLxTUEiIiIRUZCIiEhEFCQiIhIRBYmIiEREQSIiIhFRkIiISEQUJCIiEhEFiYiIRCQp1gVUh+bNm3v79u1jXYaISFxZuHDhbnfPPNV+dSJI2rdvz4IFC2JdhohIXDGzjeHsp64tERGJiIJEREQioiAREZGIKEhERCQiChIREYlIVIPEzC4zszVmlm9m91XwfoqZvRi8P9fM2gfb25vZETNbEjyeCvnM+8ExT7yXFc1zEBGRk4va9F8zSwSeAC4GtgDzzWyqu68M2e0W4HN372xm1wK/BcYG7611936VHP4Gd9d8XhGRGiCaLZKBQL67r3P3Y8ALwOhy+4wGJgbPXwGGm5lFsaawlZQ6L87fxLRl22NdiohIjRbNIGkDbA55vSXYVuE+7l4M7AeaBe91MLPFZvaBmZ1f7nPPBt1aP41W8CQYPD93Ew++tYpjxaXR+CNERGqFaAZJRb/gPcx9tgM57t4f+B4w2cwaBe/f4O69gfODx40V/uFmt5rZAjNbsGvXrtMv3ozvjujKls+P8OqiLaf9eRGRuiKaQbIFaBvyOhvYVtk+ZpYEZAB73f2ou+8BcPeFwFqga/B6a/CzEJhMWRfav3H38e6e6+65mZmnXCqmQhd0y6Rf28Y8PjNfrRIRkUpEM0jmA13MrIOZ1QOuBaaW22cqcHPwfAww093dzDKDwXrMrCPQBVhnZklm1jzYngxcCSyP1gmYGd+9uCtb9x3hpQWbT/0BEZE6KGpBEox53AVMB1YBL7n7CjN7wMxGBbs9AzQzs3zKurBOTBEeBiw1szzKBuFvd/e9QAow3cyWAkuArcDT0ToHgGFdmjOgXROeeC+fouMl0fyjRETikrmXH7aofXJzcz2S1X9nf7abrz4zl/tH9eTmwe2rrjARkRrMzBa6e+6p9tOV7WEY0rkZA9s35cn31SoRESlPQRKGE2MlOw8cZfLcTbEuR0SkRlGQhOm8Ts04t2NT/vjBWo4cU6tEROQEBclp+O6IruwqPMrzc8O6aZiISJ2gIDkNgzo2Y0jnZjz1wVoOHyuOdTkiIjWCguQ0fXdEV3YfPMZf5qhVIiICCpLTltu+KcO6ZvKnWes4dFStEhERBckZ+O6ILuw9dIyJczbEuhQRkZhTkJyB/jlNuLBbJuNnraOw6HisyxERiSkFyRm6Z0RX9h0+zsSPN8S6FBGRmFKQnKG+bRsz4qwsxs9axwG1SkSkDlOQROCeEV05UFTMs7M3xLoUEZGYUZBEoFebDC7p0YI/z17H/iNqlYhI3aQgidA9I7pSWFTMM7PXx7oUEZGYUJBEqEfrRozs1ZIJs9ez5+DRWJcjIlLtFCRV4PuXdOVocQkPvrkq1qWIiFQ7BUkV6JzVkNu/0InXFm/lw892xbocEZFqpSCpInde2JkOzdP5yZTlWmZeROoUBUkVSU1O5Fdf6s2mvYf5/YzPYl2OiEi1UZBUofM6NWNsblue/nAdK7cdiHU5IiLVQkFSxX50eXea1E/mR68tpaTUY12OiEjUKUiqWOP69fjZF3uSt2U/k+ZsiHU5IiJRpyCJgi/2acUF3TL57+lr2LrvSKzLERGJKgVJFJgZ/zW6F+7ws9eX464uLhGpvRQkUdK2aX2+f0lXZqwuYNryHbEuR0QkahQkUfS1we3p3SaDn09doUUdRaTWUpBEUVJiAr/+cm/2HjrGb6atjnU5IiJRoSCJsl5tMrhlaAf+Om8T89bvjXU5IiJVTkFSDe4Z0YXsJmn86LWlHC3W8ikiUrsoSKpB/XpJ/PKqXqzddYg/vr821uWIiFQpBUk1uaBbFqP7tebJ99aSX1AY63JERKqMgqQa/fTKHtRPSeT7L+Wpi0tEag0FSTVq3iCF33y5D3lb9vMr3QRLRGoJBUk1u6xXS74xtAMT52zk73nbYl2OiEjEFCQx8MOR3Tk7pzH3vbqUdbsOxrocEZGIRDVIzOwyM1tjZvlmdl8F76eY2YvB+3PNrH2wvb2ZHTGzJcHjqZDPDDCzZcFnHjMzi+Y5RENyYgKPX3829ZIS+Nbzi3RHRRGJa1ELEjNLBJ4ARgI9gOvMrEe53W4BPnf3zsAjwG9D3lvr7v2Cx+0h2/8I3Ap0CR6XRescoql14zQevbY/a3YW8rO/LY91OSIiZyyaLZKBQL67r3P3Y8ALwOhy+4wGJgbPXwGGn6yFYWatgEbuPsfLltSdBFxV9aVXjy90zeTbF3bm5YVbeGnB5liXIyJyRqIZJG2A0N+OW4JtFe7j7sXAfqBZ8F4HM1tsZh+Y2fkh+285xTEBMLNbzWyBmS3YtWtXZGcSRXeP6MqQzs346evLWbVdt+cVkfgTzSCpqGVR/sYcle2zHchx9/7A94DJZtYozGOWbXQf7+657p6bmZl5GmVXr8QE49Gx/clIS+Zbzy+isEirBItIfIlmkGwB2oa8zgbKz3f95z5mlgRkAHvd/ai77wFw94XAWqBrsH/2KY4ZdzIbpvCH6/qzae9h7nt1mW6EJSJxJZpBMh/oYmYdzKwecC0wtdw+U4Gbg+djgJnu7maWGQzWY2YdKRtUX+fu24FCMzs3GEu5CfhbFM+h2gzq2Ix7L+nGm8u2M/HjDbEuR0QkbFELkmDM4y5gOrAKeMndV5jZA2Y2KtjtGaCZmeVT1oV1YorwMGCpmeVRNgh/u7ufWIP9DuDPQD5lLZVp0TqH6nbbsI4M757Fg2+tYsnmfbEuR0QkLFYXulFyc3N9wYIFsS4jLPsOH+OKx2YD8OZ3htK4fr0YVyQidZWZLXT33FPtpyvba5jG9evx5A1nU1BYxPdeyqO0tPYHvYjENwVJDdS3bWN+emUPZq4u4NfTtLijiNRsSafawczSgSPuXmpmXYHuwDR31zzVKLrx3HasLTjI0x+up1VGGuOGdoh1SSIiFQqnRTILSDWzNsAM4OvA/0azKAEz42df7MmlPVvwX2+u5K1l22NdkohIhcIJEnP3w8CXgT+4+5coWztLoiwxwfj9tf05O6cJ97y4hHnr9576QyIi1SysIDGz84AbgDeDbafsEpOqkZqcyJ9vyiW7cRrfnLRAt+kVkRonnCC5B/gRMCW4DqQj8F50y5JQTdLrMXHcQJITE7h5wnx2HiiKdUkiIv90yiBx9w/cfZS7/9bMEoDd7v6daqhNQrRtWp9nv3YOnx8+xtefna81uUSkxjhlkJjZZDNrFMzeWgmsMbMfRL80Ka93dgZP3nA2a3YW8q3nF3GsuDTWJYmIhNW11cPdD1B234+3gBzgxqhWJZW6oFsWv/5ybz78bDf3vbZUCzyKSMyFM2iebGbJlAXJ4+5+3Mz02yuGrslty/Z9RTzyj09pnZHGvZd2i3VJIlKHhRMkfwI2AHnALDNrB+gOTDH2neGd2b7/CI+/l0+rxqncMKhdrEsSkTrqlEHi7o8Bj4Vs2mhmF0avJAmHmfHLq3qx80ARP319OZkNUrikZ8tYlyUidVA4g+0ZZvbwidvWmtlDQHo11CankJSYwOPXn03vNhncOXkRM1btjHVJIlIHhTPYPgEoBK4JHgeAZ6NZlIQvPSWJSeMGcVarRtz+3EL+sVJhIiLVK5wg6eTuP3f3dcHjfqBjtAuT8GXUT+Yvt5SFyR3PK0xEpHqFEyRHzGzoiRdmNgQ4Er2S5ExkpJWFSY8gTN5VmIhINQknSG4HnjCzDWa2AXgcuC2qVckZyUhLZtItg+jROoNvPb+Qd1bsiHVJIlIHhLNESp679wX6AH3cvT/QJeqVyRkpa5kMpEfrsgF4hYmIRFvYd0h09wPBFe4Aj0SpHqkCjVL/FSbfen4R0xUmIhJFZ3qrXavSKqTKnQiTXm0yuPP5Rby9XGEiItFxpkGiJVLiQKPUZCbdMpDe2RncNXkRby/XXRZFpOpVGiRmtszMllbwWAa0qMYaJQKNUpOZNO5EmCxWmIhIlTvZEilXVlsVElUNgzC5ecI87pq8mN9eXcLVA7JjXZaI1BKVBom7b6zOQiS6GqaWTQ2+7S8L+P7LeewsLOKOL3TCTMNdIhKZMx0jkTjUICWJZ782kFF9W/O7t9dw/99XUlKq4S4RiUw4y8hLLVIvKYFHx/ajRaMUnv5wPQWFRTx8TT9SkxNjXZqIxCkFSR2UkGD85IoeZDVM5cG3VrHn4DzG35RLRlpyrEsTkThUaZAEs7Mq7fdw9z5RqUiqzTeHdSSrUQr3vpzHNU/NYeK4gbTMSI11WSISZ8KZtXVn8PMvwc8bgMNRq0iq1eh+bWjeIIXb/rKQLz/5ERPHDaRLi4axLktE4kilg+3uvjGYuTXE3f/D3ZcFj/uAS6uvRIm2IZ2b88Kt53KsxBnz1BwWbNgb65JEJI6EM2srvdwy8oPRHRJrnV5tMpjyrcE0Ta/HDX+eq/W5RCRs4QTJLfz/ZeSfBMZFtSqJibZN6/PqHYPLbpD13EImfrwBd00PFpGTC2cZ+YUhy8j3dfd+7r4o+qVJLDRNr8fkbw7iou5Z/HzqCn48ZTnHiktjXZaI1GCnDBIzyzCzh4GZwAwze8jMMsI5uJldZmZrzCzfzO6r4P0UM3sxeH+umbUv936OmR00s3tDtm0I1gFbYmYLwqlDTk/9ekn86cZc7rigE3+dt4mv/nkuuw8ejXVZIlJDhdO1NQEoBK4JHgeAZ0/1ITNLBJ4ARgI9gOvMrEe53W4BPnf3zpTd4+S35d5/BJhWweEvDFpGuWHUL2cgMcH44WXd+f21/cjbso/Rj3/Eim37Y12WiNRA4QRJJ3f/ubuvCx73Ax3D+NxAID/4zDHgBWB0uX1GAxOD568Awy1Y/MnMrgLWASvCORGJjtH92vDy7edRUuqM+eMc3lyq1YNF5P8LJ0iOlJu1NQQ4Esbn2gCbQ15vCbZVuI+7FwP7gWZmlg78ELi/guM68I6ZLTSzW8OoQyLUJ7sxU789hLNaNeTOyYt4+J01lGqNLhEJhLNEyh3AxGBcxIC9wM1hfK6iZWXL//apbJ/7gUfc/WAFq9MOcfdtZpYFvGtmq9191r/94WUhcytATk5OGOXKyWQ1TOWvt57Lf05ZzmMz81m9o5CHx/ajQYpW2RGp68KZtbUkZNZWb3fv7+5Lwzj2FqBtyOtsYFtl+5hZEpBBWVANAn4XTDe+B/ixmd0V1LMt+FkATKGsC62iuse7e66752ZmZoZRrpxKSlIivxvTh59d2YN/rNrJ1U9+zKY9WuRApK473VlbM09j1tZ8oIuZdTCzesC1wNRy+0zlX62bMcBML3O+u7d39/bAo8Cv3P1xM0s3s4ZBXenAJcDyMGqRKmJmjBvagYnjBrLjQBGjnpjNx2t3x7osEYmhqM3aCsY87gKmA6uAl9x9hZk9YGajgt2eoWxMJB/4HvBvU4TLaQHMNrM8YB7wpru/HcY5SBU7v0smf7tzCM0bpHDjM/N46oO1GjcRqaPsVFcum9kSd+93qm01WW5uri9YoEtOoqGw6Dg/fHUpby3bwUXds3joK31pkl4v1mWJSBUws4XhXGYRzVlbUgc0TE3mievP5v5RPfnws11c+YfZLNr0eazLEpFqFE6Q3M6/1traCDwebBMBysZNbh7cnlfvGExCAlzz1Bz+/OE6rdMlUkeEM2srr4JZW3nRL03iTZ/sxrzx7fO5qHsWv3xzFbf9ZSH7Dx+PdVkiEmXhjJGkAFcD7Qm57sTdH4hqZVVIYyTVy92Z8NEGfv3WKlpmpPLE9WfTt23jWJclIqepKsdI/kbZUibFwKGQh0iFzIxbhnbg5dvPwx3GPPUx//vRenV1idRS4VyWnO3ul0W9Eql1+uc04c3vDOXel/P4xd9XMm/DXn5zdR8apSbHujQRqULhtEg+NrPeUa9EaqXG9evx9E25/Pjy7kxfsZPLf/8h89brVr4itUmlQRLc82MpMBRYFNxXZGnIdpGwmBm3DuvEy7efR2KCMXb8HH739mrdMEukljhZ19aV1VaF1Aln5zThre+cz3+9sZIn31/LrM928ejYfnTOahjr0kQkAifr2vrc3TdStjxKRQ+R05aeksRvru7Dn24cwLZ9RVzx2GzdG14kzp2sRTKZslbJQsqWdg9dz90J7+ZWIhW6tGdL+uc05j9eWcrPp65g5uoC/ntMH7Iapca6NBE5Tae8jqQ20HUkNZe789wnG3nwrVWkJSfy6y/35rJerWJdlogQ/nUklbZIzOzsk33Q3RedSWEiocyMG89rz3mdmvPdF5dw+3OL+MqAbH4+qqdumiUSJ072L/Whk7znwEVVXIvUYZ2zGvDqHYN5bMZnPPl+Pp+s38Nvr+7D4E7NY12aiJyCurakxpm/YS/3vpzHxj2HuWFQDveN7E5DXcQoUu2qbIkUM6tvZv9pZuOD113MTFODJWrOad+Ut+8exjeGdmDyvE1c+sgs3l9TEOuyRKQS4VzZ/ixwDBgcvN4C/DJqFYkAafUS+c8re/DqHYOpn5LE156dz70v52k1YZEaKJwg6eTuvwOOA7j7Ef7/VGCRqDk7pwlvfHsod17YiSmLtzLikQ94Z8WOWJclIiHCCZJjZpZG2QA7ZtYJOBrVqkRCpCYn8oNLu//zHvG3/mUh3/7rYvYc1F9DkZognCD5OfA20NbMngdmAP8R1apEKtCrTQZT7xrC9y/uytvLt3PxI7P4e942XRUvEmNhzdoys2bAuZR1aX3i7rujXVhV0qyt2mfNjkL+45U88rbsZ8RZWfxiVE+ym9SPdVkitUpVztp6wN33uPub7v4GsDdomYjETLeWDXn1jsH85PKz+Ch/Dxc/PIunPljL8RKtKCxS3cLp2soxsx/BP2+7+zrwWVSrEglDUmIC3xzWkX98/wsM7dKc30xbzZWPzWbBBt3vRKQ6hRMkXwd6B2Hyd+A9d/9FVKsSOQ1tGqfx9E25jL9xAIVFxxnz1Bzue3Upnx86FuvSROqESsdIyq21lQz8CfgIeAbia60tjZHUHYeOFvPYjM/48+z1ZKQl86OR3RkzIBszzVgXOV3hjpGcLEjeO8nn3N3jZq0tBUnds3rHAX4yZTkLN37OwA5N+dWXeukGWiKnKeIgqU0UJHVTaanz0oLN/Hraag4fK+ab53fkros6U7+eVhUWCUdVtEi+6u7Pmdn3Knrf3R+OsMZqoyCp2/YcPMqDb63itUVbaZWRyn0juzOqb2t1d4mcQlVM/00Pfjas4NEg4gpFqkmzBik8fE0/Xrn9PJo1qMfdLyzhmj/NYfnW/bEuTaRWOKOuLTO7x90fjUI9UaEWiZxQUuq8vGAz/z19DXsPH+Pac9py7yXdaNYgJdalidQ4VXZBYiUq7O4SqekSE4xrB+Yw894LGDekAy8v2MIF//M+E2av18WMImfoTINEncsS1zLSkvnplT14+57z6de2MQ+8sZLLf/8hH362K9alicSdMw2S2j/VS+qEzlkNmTRuIE/flMuxklJufGYe35y0gI17DsW6NJG4cbJZW4VUHBgGpLl73Myh1BiJhONocQnPzF7P4zPzOV5Syo3ntufbF3WmSXq9WJcmEhO6jiSEgkROR8GBIh75x6e8OH8zDVKSuOuiztx0XntSkxNjXZpItYr2YHu4RVxmZmvMLN/M7qvg/RQzezF4f66ZtS/3fo6ZHTSze8M9pkikshql8usv92Ha3cMY0K4Jv3prNcMf+oC/LdlKaWnt/4+XyOmKWpCYWSLwBDAS6AFcZ2Y9yu12C/C5u3cGHgF+W+79R4Bpp3lMkSrRrWVDnv36QJ7/xiAy0pK5+4UlXPXkR3yybk+sSxOpUaLZIhkI5Lv7Onc/BrwAjC63z2hgYvD8FWC4BZcbm9lVwDpgxWkeU6RKDencnDe+PZSHvtKXXYVHuXb8J3xj4gLyCw7GujSRGiGaQdIG2BzyekuwrcJ93L0Y2A80M7N04IfA/WdwTADM7FYzW2BmC3bt0pROiUxCgnH1gGzeu/cCfnBpNz5Zt4dLH53FT6Yso+BAUazLE4mpcO6QWGhmB8o9NpvZFDPreLKPVrCtfAdzZfvcDzzi7uX/yxfOMcs2uo9391x3z83MzDxJmSLhS01O5M4LO/PBDy7gq4NyeHH+Zob993v8Ztpq9h3W/U+kbgpnCu/DwDZgMmW/yK8FWgJrgAnABZV8bgvQNuR1dnCcivbZYmZJQAawFxgEjDGz3wGNgVIzKwIWhnFMkahr1iCF+0f3YtzQDjz6j8/406y1PD93I7cN68jXh3QgPSVuZseLROyU03/NbK67Dyq37RN3P9fM8ty9byWfSwI+BYYDW4H5wPXuviJknzuB3u5+u5ldC3zZ3a8pd5xfAAfd/X/COWZFNP1Xom31jgM89M6nvLtyJ83S63HnhZ25flCOpgxLXKvK6b+lZnaNmSUEj9Bf9JWmUDDmcRcwHVgFvOTuK8zsATMbFez2DGVjIvmUrd910um8lR0zjHMQiaruLRvx9E25TPnWYLq1bMgDb6zkov95n5fmb6ZYa3hJLRdOi6Qj8HvgvGDTHOC7lLUIBrj77KhWWAXUIpHq9lH+bn43fQ15m/fRsXk637ukK5f3akVCgpapk/ihK9tDKEgkFtydd1bu5KF31vDpzoOc1aoRdw/vwiU9WihQJC5UWdeWmWUHM7QKzGynmb1qZtlVU6ZI7WVmXNqzJdPuHsYjY/tSdLyE259byBV/mM3by3foKnmpNcIZI3kWmAq0puyajb8H20QkDIkJxpf6Z/PudxUoUjuFM0ayxN37nWpbTaauLalJiktK+fvSbTw2I5/1uw+py0tqrKqctbXbzL5qZonB46uAFhsSOUNJiQlqoUitEk6LJAd4nLJZWw58DHzH3TdFv7yqoRaJ1GQVtVDuvLATI3u1IlEtFImhqM7aMrN73P3RM6osBhQkEg9OBMofZuazbtchOjRP544vdOKq/m2olxTVOz6IVCjaQbLJ3XPOqLIYUJBIPCkpdd5ZsYPH38tnxbYDtM5I5dZhHRl7Tg5p9XSlvFSfaAfJZndve+o9awYFicQjd+eDT3fx5HtrmbdhL83S6zFuaAduPK8djVKTY12e1AFqkYRQkEi8m7d+L0++n8/7a3bRMCWJmwa34+tDOtC8QUqsS5NaLOIgMbNCKl5Ly4A0d4+b5U0VJFJbLN+6nz++v5a3lm8nJSmBsblt+cb5HWnbtH6sS5NaSEukhFCQSG2zdtdBnnp/La8v2UpJqTOydytuG9aRPtmNY12a1CIKkhAKEqmtduwv4tmP1zP5k00UHi1mUIem3PaFjlzQNUsXN0rEFCQhFCRS2xUWHeeFeZuZ8NF6tu8voktWA745rCOj+7UmJUkzveTMKEhCKEikrjheUsrf87YxftY6Vu8oJKthCl8b0p4bBrUjI00zveT0KEhCKEikrnF3PvxsN+NnrWN2/m7S6yXyldy2fG1we9o3T491eRInFCQhFCRSly3fup9nZq/njaXbKC51hnfPYtyQDpzXqRlmGkeRyilIQihIRKDgQBHPfbKR5+ZuYu+hY3Rv2ZBxQzowql9r3VteKqQgCaEgEfmXouMlTM3bxoTZ61m9o5Bm6fW4YVAOXz23HVmNUmNdntQgCpIQChKRf+fuzFm3hwmzNzBj9U6SEowr+7Tma4Pb07etrkeR8IMkbq5OF5GqZWYM7tScwZ2as2H3If734w28vGAzUxZvpW/bxtx0bjuu6NNK3V5ySmqRiMg/FRYd57VFW5k0ZwNrdx2iaXo9xp7TlhsG5ZDdRMuw1DXq2gqhIBE5Pe7Ox2v3MGnOBt5duROA4We14Kbz2jG0c3PN9qoj1LUlImfMzBjSuTlDOjdn674jTJ67kRfmbebdlTvpmJnOjee24+oB2VrOXgC1SEQkTEeLS3hr2XYmzdnI4k37SEtOZHS/1lw/KEeLRdZS6toKoSARqVrLtuznuU82MjVvG0eOl9CrTSOuH9iO0f1ak56ijo7aQkESQkEiEh0Hio7z+uKtTJ67idU7CmmQkvTPVkrP1hmxLk8ipCAJoSARiS53Z9GmfUyeu4k3lm7jaHEpfds25oaBOVzZtxX166mVEo8UJCEUJCLVZ//h47y6aAuT520iv+AgDVOSuKp/G8ae05ZebdRKiScKkhAKEpHq5+7M3/A5k+du5K3lOzhWXErP1o0Ye05bRvdtQ0Z9zfiq6RQkIRQkIrG1//Bx/pa3lRfnb2bFtgOkJCVwWa+WjD2nLed2aKa7OdZQCpIQChKRmmP51v28OH8zry/ZSmFRMTlN63NNbjZjBrSlZYYWjaxJFCQhFCQiNU/R8RLeXr6DF+Zv4pN1e0kwuKBbFmMGZDP8rCzdIrgG0JXtIlKjpSYnclX/NlzVvw0bdh/i5YWbeWXhFmauLiAjLZlRfVszZkA2fbIztCRLDRfVFomZXQb8HkgE/uzuvyn3fgowCRgA7AHGuvsGMxsIjD+xG/ALd58SfGYDUAiUAMXhpKVaJCLxoaTU+Sh/N68s3ML0FTs4WlxK56wGjBmQzZf6t6GF7pdSrWLetWVmicCnwMXAFmA+cJ27rwzZ51tAH3e/3cyuBb7k7mPNrD5wzN2LzawVkAe0Dl5vAHLdfXe4tShIROLPgaLjvLl0O68s3MLCjZ+TYHB+l0zGDMjm4h4ttLx9NagJXVsDgXx3XxcU9AIwGlgZss9o4BfB81eAx83M3P1wyD6pQO0fyBGR/6dRajLXDczhuoE5rNt1kNcWbeW1RVv49l8X0zA1iSv7tOZL/duQ266JZn3FWDSDpA2wOeT1FmBQZfsErY39QDNgt5kNAiYA7YAb3b04+IwD75h0acMsAAAKxUlEQVSZA39y9/GISK3WMbMB917aje9d3JU56/bwysItvL54K3+dt4nsJmlc1a9srKVzVoNYl1onRTNIKvovQvmWRaX7uPtcoKeZnQVMNLNp7l4EDHH3bWaWBbxrZqvdfda//eFmtwK3AuTk5ERyHiJSQyQk/Gt5+19eVcy7K3fy2uKtPPl+Po+/l0/vNhlc1b8NX+zbiqyGGk+pLglRPPYWoG3I62xgW2X7mFkSkAHsDd3B3VcBh4Bewettwc8CYAplXWj/xt3Hu3uuu+dmZmZGfDIiUrOkB0uvTBo3kE9+PJyfXtkDgP96YyXn/moGN0+Yx+uLt3L4WPEpjiSRimaLZD7Qxcw6AFuBa4Hry+0zFbgZmAOMAWa6uwef2Rx0d7UDugEbzCwdSHD3wuD5JcADUTwHEYkDWQ1TuWVoB24Z2oH8gkKmLN7K64u3cc+LS0hLTuTiHi0Y1bc1w7pmUi8pmv9/rpuiPf33cuBRyqb/TnD3B83sAWCBu081s1TgL0B/yloi17r7OjO7EbgPOA6UAg+4++tm1pGyVgiUheBkd3/wVHVo1pZI3VNa6izY+DmvL9nKW8u2s+/wcTLSkhnZqyWj+rZmUMdmJGqQ/qRiPv23JlGQiNRtx0tKmf3ZbqbmbeOdFTs4dKyErIYpXNGnFaP6tqZf28a66LECCpIQChIROeHIsRJmri5gat5W3luzi2PFpbRtmsYX+7Tmi31b071lQ4VKQEESQkEiIhU5UHSc6ct3MDVvGx+v3UNJqdMxM50re7fiij6t6dayYaxLjCkFSQgFiYicyu6DR5m+YgdvLt3OJ+v2UOrQJasBV/RpxRW9W9GlRd0LFQVJCAWJiJyOXYVHeXv5dt5Yup15G/biDt1aNOSKPq24vHerOnPho4IkhIJERM5UwYEipi0va6nM31gWKt1bNuSyXi25vHcrumQ1qLVjKgqSEAoSEakKO/YXMW35dt5atp0FGz/HHTpmpjOyV0tG9mpFz9aNalWoKEhCKEhEpKoVHChi+oodTFu+g7nr91JS6rRtmsZlPVsysncr+mU3jvvFJBUkIRQkIhJNew8d492VZaHyUf5ujpc4LRulclmvllzasyXntG9CUmL8XVGvIAmhIBGR6rL/yHFmrt7JtGU7+ODTXRwtLqVJ/WSGn9WCS3q0YFjXzLi5l4qCJISCRERi4dDRYmZ9uovpK3YwY3UBhUXFpCUn8oWumVzSswXDu7cgo35yrMusVE24sZWISJ2WnpLEyN6tGNm7FcdLSvlk3R7eWbGTd1bu4O0VO0hKMAZ1bMqlPVtycY8WtMpIi3XJZ0QtEhGRalZa6izdup/pK3YwfcUO1u06BEDvNhmMOKsFI3pk0aNV7GeAqWsrhIJERGqy/IKDvLNyBzNWFbBoU9m04tYZqYzo0YLhZ7Xg3I5NSUmq/nEVBUkIBYmIxIvdB48yc3UB/1i5kw8/282R4yWk10vkC90yGXFWCy7slkWT9HrVUouCJISCRETiUdHxEj5eu5t3VxYwY9VOCgqPkmCQ264pF52VxUXds6J6Zb2CJISCRETiXWmps2zrft5duZOZqwtYuf0AANlN0hjePYsLu2dxbsdmVTq1WEESQkEiIrXN9v1HeG/1Lmau3sns/N0UHS8lLTmRIZ2bMzxorbRolBrRn6EgCaEgEZHarOh4CXPW7WHmqgJmri5g674jAPRs3YhJ4wbSrEHKGR1X15GIiNQRqcmJXNgtiwu7ZfGAO5/uPMjM1QUs2fw5TathYF5BIiJSi5gZ3Vo2rNa7O8bfKmIiIlKjKEhERCQiChIREYmIgkRERCKiIBERkYgoSEREJCIKEhERiYiCREREIlInlkgxs13AxjP8eHNgdxWWE2u17Xyg9p1TbTsfqH3nVNvOByo+p3bunnmqD9aJIImEmS0IZ62ZeFHbzgdq3znVtvOB2ndOte18ILJzUteWiIhEREEiIiIRUZCc2vhYF1DFatv5QO07p9p2PlD7zqm2nQ9EcE4aIxERkYioRSIiIhFRkFTCzC4zszVmlm9m98W6nqpgZhvMbJmZLTGzuLxlpJlNMLMCM1sesq2pmb1rZp8FP5vEssbTUcn5/MLMtgbf0xIzuzyWNZ4OM2trZu+Z2SozW2Fmdwfb4/k7quyc4vJ7MrNUM5tnZnnB+dwfbO9gZnOD7+hFMwv7jljq2qqAmSUCnwIXA1uA+cB17r4ypoVFyMw2ALnuHrfz381sGHAQmOTuvYJtvwP2uvtvgtBv4u4/jGWd4arkfH4BHHT3/4llbWfCzFoBrdx9kZk1BBYCVwFfI36/o8rO6Rri8HsyMwPS3f2gmSUDs4G7ge8Br7n7C2b2FJDn7n8M55hqkVRsIJDv7uvc/RjwAjA6xjUJ4O6zgL3lNo8GJgbPJ1L2jzwuVHI+ccvdt7v7ouB5IbAKaEN8f0eVnVNc8jIHg5fJwcOBi4BXgu2n9R0pSCrWBtgc8noLcfwXJ4QD75jZQjO7NdbFVKEW7r4dyv7RA1kxrqcq3GVmS4Our7jpBgplZu2B/sBcasl3VO6cIE6/JzNLNLMlQAHwLrAW2OfuxcEup/U7T0FSMatgW23oAxzi7mcDI4E7g24VqXn+CHQC+gHbgYdiW87pM7MGwKvAPe5+INb1VIUKziluvyd3L3H3fkA2ZT0wZ1W0W7jHU5BUbAvQNuR1NrAtRrVUGXffFvwsAKZQ9heoNtgZ9GOf6M8uiHE9EXH3ncE/9FLgaeLsewr63V8Fnnf314LNcf0dVXRO8f49Abj7PuB94FygsZklBW+d1u88BUnF5gNdglkM9YBrgakxrikiZpYeDBRiZunAJcDyk38qbkwFbg6e3wz8LYa1ROzEL9zAl4ij7ykYyH0GWOXuD4e8FbffUWXnFK/fk5llmlnj4HkaMIKycZ/3gDHBbqf1HWnWViWCqXyPAonABHd/MMYlRcTMOlLWCgFIAibH4zmZ2V+BCyhbqXQn8HPgdeAlIAfYBHzF3eNiALuS87mAsu4SBzYAt50YX6jpzGwo8CGwDCgNNv+YsjGFeP2OKjun64jD78nM+lA2mJ5IWWPiJXd/IPgd8QLQFFgMfNXdj4Z1TAWJiIhEQl1bIiISEQWJiIhEREEiIiIRUZCIiEhEFCQiIhIRBYlIFTCzkpBVYJdU5YrRZtY+dHVgkZom6dS7iEgYjgRLTojUOWqRiERRcA+Y3wb3f5hnZp2D7e3MbEaw4N8MM8sJtrcwsynBvSLyzGxwcKhEM3s6uH/EO8EVySI1goJEpGqklevaGhvy3gF3Hwg8TtlqCQTPJ7l7H+B54LFg+2PAB+7eFzgbWBFs7wI84e49gX3A1VE+H5Gw6cp2kSpgZgfdvUEF2zcAF7n7umDhvx3u3szMdlN2s6Tjwfbt7t7czHYB2aFLUwRLl7/r7l2C1z8Ekt39l9E/M5FTU4tEJPq8kueV7VOR0DWPStD4ptQgChKR6Bsb8nNO8PxjylaVBriBstudAswA7oB/3nyoUXUVKXKm9L8akaqRFtxx7oS33f3EFOAUM5tL2X/crgu2fQeYYGY/AHYBXw+23w2MN7NbKGt53EHZTZNEaiyNkYhEUTBGkuvuu2Ndi0i0qGtLREQiohaJiIhERC0SERGJiIJEREQioiAREZGIKEhERCQiChIREYmIgkRERCLyf+IQEY2Th5UEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epoch_cost)\n",
    "plt.ylabel('Log Likelihood Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 96.00%\n"
     ]
    }
   ],
   "source": [
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32, device=device)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.float32, device=device)\n",
    "\n",
    "test_acc = model1.evaluate(X_test_tensor, y_test_tensor)\n",
    "print('Test set accuracy: %.2f%%' % (test_acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAADFCAYAAAAMsRa3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX5+PHPmSWZhAARwpqwg+yEQEhC3epWsO67gmFf2mq1am1daqXWWi3WnwutEhJWQaVfLfrVVmpLW1u/kBB2EBQEgQRkX5NMMknO74+AhjAzmWTuzL0z87xfr7xeZpLc+xDnyXPuuc89R2mtEUIIISKFzewAhBBCiKaQwiWEECKiSOESQggRUaRwCSGEiChSuIQQQkQUKVxCCCEiihQuIYQQEUUKlxBCiIgihUsIIUREcZhx0pSUFN29e3czTi2EIdasWXNYa93O7DjOkpwS0SDQvDKlcHXv3p3i4mIzTi2EIZRSu82OoT7JKRENAs0rmSoUQggRUaRwCSGEiChSuIQQQkQUU+5xCSFCz+PxUFJSgtvtNjsUS3G5XKSlpeF0Os0ORTRT1BauyuoaHn1nExO+0530LslmhyNE2JWUlNCyZUu6d++OUiro49Vqzb5jFSS3iCMpPjL/dGitOXLkCCUlJfTo0cPscEQzRe1U4YlyD6u/Osq4uUVs3X/S7HCECDu3203btm0NKVoAtbWasqoavjpcRnlltSHHDDelFG3btpWr0AgXdOFSSrmUUkVKqQ1KqS1KqV8ZEViw2rdysWRKDglOO/fkF7Lj4GmzQxIiIEbmlFFFC8Bht9EzpQUOu2LXkTIqqiK3eInIZsQVVyVwhdY6HRgKjFZK5Rhw3KB1bZvI4qnZKKUYm7+K3UfKzA5JiEBYNqecjrriZVOKXYfLcHtqzA5JxKCgC5euc/ZyxnnmQwd7XKP0apfE4inZVFXXMmZOIaXHK8wOSQi/rJ5TcQ47PVNaoJRi56EyKptYvGbMmMELL7wQktjWrFnD4MGD6d27N/fffz9aW+bXJgxkyD0upZRdKbUeOAh8rLUu9PI905RSxUqp4kOHDhlx2oD17diSRZOzOen2MGbOKg6clPltYW1m5NSydaVc9NwKejz6IRc9t4Jl60p9fm+8006PlBYA7DxcRlW1Na68fvjDH5KXl8f27dvZvn07H330kdkhiRAwpHBprWu01kOBNCBLKTXIy/fkaa0ztdaZ7dqFf4m3QamtWTApi8OnKhkzZxWHT1eGPQYhAhXunFq2rpTH3t1E6fEKNFB6vILH3t3kt3i5zhSvWq3PFK/a875n4cKFDBkyhPT0dHJzc8/7+pw5cxgxYgTp6enceuutlJeXA/CnP/2JQYMGkZ6ezqWXXgrAli1byMrKYujQoQwZMoTt27efc6z9+/dz8uRJRo4ciVKKcePGsWzZsiB+K8KqDO0q1FofB/4FjDbyuEYZ1vUCCiaMoPR4BffkF3K8vMrskITwK1w5NXP551Q0mPKr8NQwc/nnfn8uIa6ueNXUaHYdLsNT823x2rJlC7/5zW9YsWIFGzZs4OWXXz7v52+55RZWr17Nhg0b6N+/PwUFBQA8/fTTLF++nA0bNvD+++8D8Prrr/PAAw+wfv16iouLSUtLO+dYpaWl57yWlpZGaanvwisilxFdhe2UUsln/jsBuArYFuxxQyWnZ1vycjPZeaiMcXOLOOn2mB2SEOcwI6f2+bj36+v1+hLjHHRPaYGnppZdh8uoPlO8VqxYwW233UZKSgoAbdq0Oe9nN2/ezCWXXMLgwYNZvHgxW7ZsAeCiiy5iwoQJzJkzh5qauoI6cuRInn32WZ5//nl2795NQkLCOcfydj9LOgijkxFXXJ2AfyqlNgKrqZuP/8CA44bMpRe2449jh/HZvpNMmrea8ght6xVRK+w51Tk5oUmvN9Qi3kH3tolUVp8pXrW1aK0bLRwTJkxg1qxZbNq0iaeeeuqb56tef/11nnnmGfbu3cvQoUM5cuQIY8aM4f333ychIYFRo0axYsWKc46VlpZGSUnJN5+XlJTQuXPngOIXkcWIrsKNWusMrfUQrfUgrfXTRgQWalcN6MDLd2Wwds8xpiwolrZeYRlm5NQjo/qS4LSf81qC084jo/oGfIwkl5NubRJxV9fy1eFyvnv5FSxdupQjR44AcPTo0fN+5tSpU3Tq1AmPx8PixYu/ef3LL78kOzubp59+mpSUFPbu3cvOnTvp2bMn999/PzfccAMbN24851idOnWiZcuWrFq1Cq01Cxcu5MYbb2zKr0FEiKhdOSMQ1w7pxAu3p7Ny5xF+8MYaKi3SGSVEuN2UkcpvbxlManICCkhNTuC3twzmpozUJh2nVYKTrm0SqKiqoUXH7jz22ONcdtllpKen89BDD533/b/+9a/Jzs7m6quvpl+/ft+8/sgjjzB48GAGDRrEpZdeSnp6Om+//TaDBg1i6NChbNu2jXHjxp13vNdee40pU6bQu3dvevXqxTXXXNPk34WwPmXGcw6ZmZnaSpveLSncw+N/3sSogR2YNWYYTntM13MRAKXUGq11ptlxnOUtp7Zu3Ur//v1Nied4eRV7jpaTFO+ge9sW2GzWutdk5u9G+BZoXslfaGBMdld+ed0Alm85wMNLN1BTKw8tChGM5MQ40i5I5HRlNXuOllMrDwILA0XmEs8hMOniHrira/jdR58T77Dx/K1DLDdKFCKStGkRV7ei/PEK9h4tp2ubROnyE4aQwlXPj77bG7enllf+sR2X087TNw6URBMiCClJ8WgN+09UUHKsgrQLEiSnRNCkcDXw4FV9cHtqyPtkJy6njce/318STYggtGsZT63WHDjpRqm6xg/JKREMKVwNKKV47Jp+uD01zPnPLhKcdh76XuAtwUKI87VvGY/WmoOnKrEpRafWLileotmkcHmhlGLG9QNxe2p4ZcUO4p127r28t9lhCRGxlFJ0aOWiVsPh05UoBR1bSfESzSNdhT7YbIrf3jKEG4d2Zubyzyn47y6zQxIioqkzV1rzX53JCzNf4OAp4xe6fuKJJ+jSpQtJSUmGH1tYh1xx+WG3KX5/ezqVnlp+/cFnuJw2xmZ3MzssIYz3bCpUedklPC4JHjduoVqlFC1dDmrsdg6cdGNT0K6ly7DjX3/99dx333306dPHsGMK65ErrkY47DZeuTuDy/u24xfLNvPOmpLGf0iISOOtaPl7PUDetjVRStE6wUnrBCezXptNxvBMQ7Y1AcjJyaFTp05BxSysT664AhDnsPHaPcOZvGA1j/zPBuKdNq4bIot3CuHP2W1NPv30U1JSUs5Zq1ApRZc2idx0083cOmY8aRck8uJvn6agoIAf//jH32xrkpqayvHjx4FvtzUZO3YsVVVV36waL2KPXHEFyOW0M2dcJsO7XcBP3lrP37Z8bXZIQlhaY9ua2JTieOlOJt/2fbKHD2XRG28Eta2JiB1SuJogMc7B3AkjGNi5FfctWce/vwh+u3QholUg25pMmjSRvNf+yEf/KWLq/T/jZFndVGFztjURsUMKVxO1dDlZMCmLXu2TmLawmJVfHjE7JCEs6corrwxoW5PU1M6ktorjr+/9D2WV1Zys8DRrWxMRO6RwNUNyYhxvTM6iS5tEJi9YzZrdx8wOSYjgxPloH/f1egAGDhzIE088EdC2JqNHfY+hgwdgV4rdR8t58KGHm7Wtyc9+9jPS0tIoLy8nLS2NGTNmNDt+YV2yrUkQDp50c8fslRw5XcWSqTkMTmttdkgiTGRbk9Corqll5+Eyqqpr6ZHSghbxoekfi8TfTSwI27YmSqkuSql/KqW2KqW2KKUeCPaYkaJ9KxeLp+bQKsFJ7txCtn190uyQRBSI5Zxy2G30SGmB027jq8NllFdVmx2SsCAjpgqrgYe11v2BHOBepdQAA44bEVKTE1gyNZt4h4178gv58lBwz70IQYznlPNM8bLbFbsOl1EhxUs0EHTh0lrv11qvPfPfp4CtQNP2+45w3dq2YPGUHADGzilkz5FykyMSkczInDLjVoAR4hw2eqa0wKYUuw6X4/YY98xWpP5OxLcMbc5QSnUHMoBCL1+bppQqVkoVHzoUfW3kvdsnsWhyNu7qGu6es4p9xyvMDklEgWByyuVyceTIkYj9Qx3nsNMzpQUo2Hm4jEoDipfWmiNHjuByGbfMlAg/w5ozlFJJwL+B32it3/X3vdHSnOHNppITjJmzirZJcSydPpL2rSRBolE4mjOCzSmPx0NJSQlutzuEUYaep6aWw6cqUUqR0jIOhy248bbL5SItLQ2n02lQhMIogeaVIS07Sikn8A6wuLEEi3aD01ozf9IIcguKGJtfyFvTcmibFG92WCLCGJFTTqeTHj16GBuYSTaX1g0IkxPrBoQdW8uAMJYZ0VWogAJgq9b6xeBDinzDu7Uhf3wme46Wk1tQxIlyj9khiQgiOXW+QamtWTApiyOnKxmTv4pDIdgSRUQOI+5xXQTkAlcopdaf+fi+AceNaN/plcLs3OHsOHiacfOKOOWW4iUCJjnlRUbXC5g7YQT7jleQW1DIsbIqs0MSJjGiq/C/WmultR6itR565uMvRgQX6b7btz2zxmSwufQEk+cXyzMpIiCSU75l92xL/rgR7Dxcxri5RZyUAWFMkm1NQux7Azvy0p1DeeCtdUxbuIb88Zm4nHazw2pcoBsLhmkDQiHOurhPCq+NHcb0RWuYOG81CydlhWyFDWFN8n87DK5P70xldS0//dMGfrR4La/fM5w4h8WXiQx0Y8EQbUAohD9X9u/Aq3dncO+StUxesJr5E7MiY0DYkK+Bny8yIARkkd2wuW14Gs/cNIgV2w7ywFvrqK6pNTskISLaNYM78eIdQyncdZTpi9ZQWR2BG0s2dYAnA0JArrjC6p6cblRW1/LrDz7jp3/awO/vGIrd5n+/IiGEbzdlpFJZXcPP39nEfUvW8cexw3Dao3w8PsPLYt4xdiUW5f+HrWfyxT14ZFRflq3fxxN/3kRtbWSuaiCEVdw5oiu/umEgH392gAffXk9NLOZUjF2JyRWXCe69vDduTw2vrthBvMPGjBsGNrpTrKU0dV5eiBAb/53uuD01/Pav24h32Jl52xBsMpsRtaRwmeShqy+koqqG/P/uwuW08+g1/axVvOKSfHcLBlK0gtiA0DDS8RhTpl/WiwpPDS/9fTsup41nbhpkrZwShpHCZRKlFE9c2x93dQ2zP9mJy2nnwasvNDusb/n7w+5tjv2br51o/NjhKij+Oh7r/xukkEWNB67sQ4Wnhtn/rsupX1zb39rFK9CBoDiHFC4TKaV4+oZBuD21vPyP7bicdn743V5mhxV6Vmuhlz8cUUMpxaOj+1HpqaXgv7tIcNr56ai+ZoflW1MHTP4GjTFECpfJbDbF87cOobK6luc/2kaC08aEi6JjYVQhzKCU4pfXDcDtqWHWP3fgctq474o+ZodlDH9T+DFECpcF2G2KF+9Ip9JTw4z//Yx4p527s7qaHda5pCFDRBCbTfGbmwdTWV3LC3/7ApfTzpRLepodVvBkShuQwmUZTruNV8dkMG3hGh7/8yZcThs3Z6SFP5BgClSMjfqEtdltipm3DaGyuoZnPtxKvNNObk43s8MSBpDCZSHxDjuzc4czcd5qHl66gXiHne8P7hTeIJpTtAJpyAj4WA3m8JvbOPFsE3e6P3teadSIKg67jZfuzKDSs4Ynl23G5bBxe2YXs8MSQZLCZTEup5388ZmMm1vE/W+uI95h48r+HcwOK3CBdAw2pZOqfgdgU4pKc68aZTo06sQ5bPxh7DCmLCjm5+9sJN5p54b0zuENQh7NMJSsnGFBLeIdzJs4ggGdW/HDN9byn+2HzA4pcIF0DD5eWneVVv8jmGM3lZFXiCIiuJx28sYNJ7NbGx58ez3Lt3wd3gCs1kkb4eSKy6JauZwsnJTFXXmrmLqwmAUTs8ju2dbssLwLtEXX36hTiBBLjHMwd+II7skv5L4la5kzLpPv9m1vdliiGQy54lJKzVVKHVRKbTbieKJOcmIcb0zJJjU5gUnzV7NuzzGzQwqOjDoDJjkVGknxDhZMzKJP+5ZMX7SG//vysNkh1Q3oZrQ+/6Op92ljiFFThfOB0QYdS9STkhTPkqk5pLSMZ/zcIjaXhniaS65+rGI+klMh0TrRyRtTsunaJpEpC4op/uqo8SdpWIz8kQFdkxlSuLTWnwAh+L8vADq0crF4SjYtXU5yCwr54sCp0J2s4f2ncDG6YPo63tnXG/u6ySSnQqtNizgWT82mQysXE+etZmPJcWNPIEUnpJTWxmwBoJTqDnygtR7k4+vTgGkAXbt2Hb57925DzhtLvjpcxh2zV1KrYen0HHq2C8Ef2VA9aNxYJ+GME81fzsaEziyl1BqtdWaIz9EdyamQ2ne8gjtmr+SUu5q3puXQv1MrYw4c6Hs5kLyIIYHmVdi6CrXWeVrrTK11Zrt27cJ12qjSPaUFS6Zmo7VmbH4he4+WG38SI4pWw47BGSdCW1hidHQrORW8zskJvDk1hwSnnXvyC9lxMAzvpXDlRRSTdvgI07t9SxZNzqa8qoYx+avYf6LC7JDO5W+qzeLTcyI2dWmTyJKp2SilGJu/it1HyswOKbSioBlE2uEj0IDOrVg4KYux+YWMnVPIW9NzaN/SFb4AzhYab1c6/h4YltGlsKie7ZJYPCWbu/JWMmZOIUt/MJLU5ITwnDzcC+dGQTOIUe3wbwIrgb5KqRKl1GQjjit8S++SzLyJI9h/wk1ufhFHy6rCd/Kq04G9ySMoEaxGcir8+nasm8046fYwZs4qDpx0N/9gTZld8PZAvkwj+mXIFZfW+m4jjiOaZkT3NhSMz2TC/NXkFhSyZGoOrROcZofVdLLy/Hkkp8wxKLU1CyZlkZtfyNj8Qt6alkNKUnzTDyRFJ6TkHleE+07vFGbnDueLA6eYMK+I05XVwR3QjPtNwRYtuUcmDDSs6wUUTBhBybFycguKOF4extkMERApXFHg8r7tefXuYWwsOcGk+aupqKpp/sHqT1tYmUypiBDK6dmWvNxMvjx4mvFzizjp9pgdkqhHCleUGD2oIy/ekc7qr44ybVExbk8zildTnvYXIspdemE7/jh2GFv2nWTSvNWUVwU5m2EVUdDdK12FUeTGoams+vIIb67eS78nP6Jzaxc/G92PmzICbHM1+j5TBCWCEN5cNaADL9+VwY/fXMuUBcXMnTACl9NudljBiYIZCilcUWTZulKWrd/3zef7Trh59J2NAIEXr2BZfYpRiCa6dkgnKqvTefhPG/jBG2uYnTuceEeEF68IJ4Urisxc/jkVDaYI3dW1/O6jbcYWLqOfO2nKxpLBnEeIZrplWBpuTy2P/3kT97+5jj+MGYbDbtCdlqZsMikbUgJSuKLKvuPeV9HYd8KN1hqllDEn8pYgZxOq4b2xQBLq8dKmtcTHUIIK6xiT3RW3p4anP/iMh5Zu4P/dORS7zYCcasoDwVHw8LARpDkjinT286T/r/73M4xaUNmrYBPKrFXphWiCSRf34Gej+/L+hn08+s5GamtDmFMQUcswhVPUX3EtW1fKzOWfs+94BZ2TE3hkVN/w3e8Js0dG9eWxdzedM13octjI7tmW+f/3FQlxdn42qq/vK69Apuxkmk7EuB99tzduTy2v/GM7Lqedp28caNxsRkMxdiUVqKguXMvWlZ7zh7z0eAWPvbsJCGOzQhid/Tc1LNQ3Du3ME8s289q/viTBaef+K/t4P0C4p+D8zdf7U386Msbm9oU1PHhVH9yeGvI+2YnLaePx7/f/pnjF0mDZLFFduLw1K1R4api5/POofSPdlJHq9d/2zI2DcHtqePHjL3A5bUy7tFf4gvL2TJi/q7uq04E3bMiIVJhAKcVj1/TD7alhzn92keC089D3+po7WI6hxo2oLlw+mxV8vB5u4RyZ2WyK3906hMrqWp79yzZcTjvjRnYPybkC0ljBaZho8kC0sBilFDOuH0ilp5ZXVuwg3mlnSeGepg+Wm9JV66+jN4YaN6K6cHVOTqDUS5Hy18QQLmaMzBx2Gy/dOZSq6lp++d4WXA47d4zoYszBm9rSLkQUsNkUz94yGHd1XXHyxe9g+fHSwAdm/q6cQj24s9AVXVR3FT4yqi8JDZ5yT3DaeWRUX5Mi+pa/acxQctptzBqTwSV9Uvj5uxt5b71Bb7jHS6VxQ8Qku03x+9vTGT2wo8/vaXSwHAnLMFnoii6qr7h8NStY4f6WmdOY8Q47ebmZTJxfxENLNxDvsDF6UKfgDyxXXCJGOew2Xrk7g5v+8Cmf7T95ztcCGixH2T2oUIvqwgW+mxXMZvY0ZkKcnfzxIxhXUMiP31xHXq6dy/u1P/8bQzk94G96seEuyuHeJVaIJopz2Hj3R9/hhln/5YsDde/VVAsNlqOJIYVLKTUaeBmwA/la6+eMOG408/bMVbinMZPiHcybmMW1r/yHifNXA14SLVTTAw0Ln6/5+bPnibERqeRU8MxoS3c57Sy79yLGzy1i3Z7jzLhhIFcP6BDSc34jhgZ3QRcupZQd+ANwNVACrFZKva+1/izYY0czq0xj/nPbQQ6fqvzm85A1ichqGAGTnAqemW3piXEO5k4YwT35hdy7eC1zxmdy2YXtQnpOIKYGd0Y0Z2QBO7TWO7XWVcBbwI0GHDfq3ZSRyqePXsGu567l00evMGU6Yebyz3FX157zWjiaRIRfklNBMqv56ayWLicLJmXRq30S0xYWs/LLI2E5b0hZqIHEiKnCVGBvvc9LgOyG36SUmgZMA+jatasBpxVG8NUM4u3+W6NiaKoixCSngmSFZziTE+N4Y3IWd+atYvKC1SyanM3wbheE7fyGs9AVnRGFy9siXeetPKm1zgPyADIzM0O8MqX5rLbsi694fDWJ2BRsP3AKH4tDeWehN3aEk5wKktnNT2e1TYpnyZRs7pi9kglzi1gyNYfBafIwfbCMmCosAeo/xZoG7PPxvTHh7Px66fEKNN/Ory9bZ84fdn/xeHvWLd5hIynewZj8QnbZe3o/qNFXURaahrAAyakgWekZzvatXCyemkOrBCe5cwv5/OtTYY8h2qhgt7pQSjmAL4ArgVJgNTBGa73F189kZmbq4uLioM4bDs29arrouRVeR3upyQl8+ugVoQg1qHi8/TsHdm7FnXmrcDlsvD19JF3aJJ77wxZ6it4MSqk1WuvMEB07anMqnKw267H7SBl3zF5JTa3m7ekj6dUuJgdlfgWaV0FPFWqtq5VS9wHLqWvdnesvwSJFMF1JzZ1fD1WiNRaPr2fdFk3O4u68VYzNL2Tp9JF0bO369osWeoo+2kRrToWb1Z7h7Na2BYun5HBX3krGzqnLqa5tExv/QXEeQ5Z80lr/RWt9oda6l9b6N0Yc02zBdCX5mkf3N78eyunF5sQDMLBzaxZOzuZoWRVj8ldxqF7bvAitaMwpAb3bJ7Focjbu6hrG5K+yzILfkSaq1ypsrmXrSn121QXyRmvO/Hoo23eDme8f2iWZuRNGsP+4m9yCQo6VVQUdjxCxrH+nViyalM2Jcg9j5qzi4Em32SFFHClcDZy98vElkK6kmzJS+e0tg0lNTkBRdy/pt7cM9jttEcr23ebEU19WjzbMGZfJzsNljJtbxIkKT9AxCRHLBqe1Zv6kERw8VcnY/EKOnJbZjKaI+rUKm8rblc9ZTelKaur8eqjbd4Od77+4Twqv3zOM6YvWMHFeEYt0PC2UJJvwz2oNElYyvFsbCsaPYMK8InILinhzag6tE51mhxURLHvFtWxdKRc9t4Iej37IRc+tCFsrub8rnKZcpTSVt+k8Rd29rnD++/25ol8HXrkrgw0lJ5hc/Shu7SXJYrN9XXhhtcdCrGhkr7bkjctkx8HTjJtXxCm3zGYEwpKFy8w3vK8rnNTkhJCOFOtP50Fd0Tr7oIKVEv6awZ34/e3pFNb2ZVq3v1D5i6N16xCe/YiBVngRGLOXXYoUl13YjlljMthceoLJ84spr6o2OyTLs2ThMvMNb+aDi2fXLkxNTjhvmQQrJfxNGak8d8tgPvniEPctWYenprbxHxIxxwrLLkWK7w3syEt3DqV491GmLVyD28ftClHHkoUrFG/4QKceg21kMEIkJPydI7ryqxsG8vFnB3jw7fXU1MqKQ+JczX0MI1Zdn96Z392Wzn93HOZHi9dSVS0DQl8s2ZxhdKNCUx8mDseDi/VvWrdOcKIUHC/30Dk5geREJ8fKz5/rtlrCj/9Od9yeGn77123EO+zMvG0INpu3ZfZELArXnnOR0gASSJy3DU/D7anhF8s288Bb63j17gwcdkteX5jKkr8Ro6frjJp6NKphpOE9vOMVHo6Ve765n3es3IO9QQEwa521xky/rBcPXnUh76wt4cn3NhPsEmIieoRj9iJSGkCaEuc9Od148roB/HXz1/z0TxtkNsMLS15xGb3JohFTb0ZuTOev5f6smlrNBWeuvOxKnVNorTaavP/K3lR4anj9318S77Dz5HX9UUquvEToZy98DUofXrqBB99eb5krMH+DZ2+xTb64B+4zX3c57Tx782CZzajHkoULjH3DGzH12NQ3nj+BFkyt6660zNjFtSmUUvx8dF/cnhrmfrqLxDg7P7Xg1aGIPr5yqebMlb9VcqY5g+d7L++N21PDqyt24HLaeer6ATIgPMOSU4VGM2LqsTlvPF9Ti4EWzOMVnohpJ1ZK8dT1A7g7qwuz/rmDWSu2mx2SiAGB5JIVcqa5jSoPXX0hUy7uwfz/+4rnPtomU/FnWPaKy0hGTD029arN39Sit5vWTVG/WFrpxrRSimduGozbU8sLf/sCl9POlEt87OclhAECzaVAZzlClU/NbVRRSvHEtf1xV9cw+987SXDa+clVFwYdT6SLicIFwU89Xt6vHYtX7Tnn+Sp/bzx/U4tn9+Q6myAJThvlnnNbXxOcdlxOm9/uQiPvuxnFblPMvG0IldU1PPPhVuKddnJzupkSi4h+DQelNqW+mSasL5Ars1DmUzCDZ6UUT98wCLenlpf+vh2X084PLusVVDyRLmYKVzCWrSvlnTWl5xQtBdw63Hcx9LWkAuCPAAARwElEQVS6/NnXGxZSbyM9wO8ozcj7bkZy2G28dGcGVdVreHLZZlwOG7dndmn8B4Vohvq51LD4QOC3BUKdT8EMnm02xfO3DqGyupbn/roNl8PGhIt6BB1TpJLCFQBvb2gN/HPbIZ8/Y/cx8rP7uLnq703ta5Rm5QeV4xw2Zo0ZxtSFxfz8nY3EO+3ckN7Z7LBElAvmysbK+QR1sxkv3pFOpaeGGf/7GfFOO3dndTU7LFMEVbiUUrcDM4D+QJbWOir3Dm/OG9pb0fL3ui/+Cpqv+242pejx6Iem3/NyOe3k5WYyfl4RD769nji7jdGDOpoSS6SIlZwKpeZe2YR6hwYjOO02Xh2TwbSFa3j8z5twOW3cnJFmdlhhF2xX4WbgFuATA2KxrOZ0BKX6+Fr914N9oNlbtyTUFcf6Dzn+YtkmU1baB0iIszN3wggGp7bmx2+u5V+fHwzbuSNUROWUWbs4hIJRCx+E+ncS77AzO3c4OT3a8vDSDfxl035Djx8JgipcWuutWmvr9WYbrDlv6MZ+xogn/huuTOBtGrLCU8PiVXtMXVkgKd7BgklZXNihJdMXreH/dhwO27kjTSTlVKSsWhEoI1b6CNfvxOW0kz8+k2FdL+D+N9fxj60HDD2+1SkjngtQSv0L+Km/aQ2l1DRgGkDXrl2H7969O+jzhpOvNll/7bP+vnbRcyu8TkukJid803XYVD0e/fC8VeV9CeY8zXW0rIq78lZScqyChZOyyOzeJqznN5JSao3WOjOEx/8XFs+pULyH/bHSox++hPt3ctLt4Z78QrbtP0XBhEwu6dPO8HOEU6B51eg9LqXU3wFvNyae0Fq/F2hAWus8IA8gMzMz4p6i8zZv3lj7rL+5dl/3x85uHNmcpPQ1R9+U84dSmxZxvDElmztnr2LivNUsnprNkLTksMdhtmjJqVA1MwTSYWuFRz+8CXeDRyuXk4WTsrgrbxVTFxazYGIW2T3bhuRcVtLoVKHW+iqt9SAvHwEnWLTy1T77k7fXNzq37e/+WHOnF3ztotzU84dS+5YuFk/JpnWik9yCIrbuP2lKHGaKlpwKxbYlvqbaZry/JSJWkTFjK5fkxLoBYWpyApPmr2bdnmMhO5dVxMSST6HibxTVWPHx1VhxVnOS0tsc/dicrqZtjOlL5+QE3pyaQ4LTzj35hew4eMq0WETzhWLTVV+DweMV3re0t0qr+llmbUSbkhTPkqk5pLSMZ/zcIjaXngjp+cwWVOFSSt2slCoBRgIfKqWWGxNWZGhsFOWv+NQvMr40JynP7qK867lr+fTRK3jmpsGmb4zpTZc2iSyZmo1SijFzCvnqcJmp8VhFJOWUUc0M9TvwAp3qPstKrepg7ka0HVrVzWa0dDnJLSjkiwPROyA0pDmjqTIzM3VxceQ/nuLtKf2GFLDruWv9HifcN3St5POvT3FX3koS4xws/cFIv4XcSkLdnNFUkZhT3vJHgdcGowsSnbg9teetiGGFQZjVfHW4jDtmr6RWw9LpOfRsl2R2SAELNK9kqjAIgVw1BTIi9DW9cHm/dgE/DxKpz9P07diSRZOzOen2MGbOKg6cdJsdkggTXyvSNLwvm+C089T1Ay05c2BF3VNasGRqNlprxuYXsvdoudkhGU6uuAzia420QJOrYSfV5f3a8c6a0oCOF+y5rWDtnmPk5hfSsbWLt6ePJCUp3uyQ/JIrruD5e3wjNTnB0m3vkeCzfSe5e84qWiU4WDp9JJ1aW382I9C8ksJlIKOeM1m2rpSHl27wujyUt+nDaJlqXLXzCBPmFdG9bQvempZDcmKc2SH5JIUreNHyvrWyDXuPMza/kPYt43lreg7tW7rMDskvmSo0QcPGiOYWrcfe3eRzTUNvDRtWXxw0UDk925KXm8nOQ2WMm1vESbf3TjIRHczqwIsl6V2SmTdxBPtPuMnNL+JoWZXZIRlCCpfFeJv3r8/bPbNQPjsS7ntnl17Yjj+OHcZn+04yad5qyiqrQ3o+YR4zO/BiyYjubSgYn8muI2XkFhRywsejBZFECpfF+LtK8jUaDdXI1ay16K4a0IGX78pg7Z5jTF1YjLuZO0UL6zNilkI07ju9U5idO5wvDpxiwrwiTkf4gFAKl8X4ukqyK+VzNBqqkau/jfVC7dohnfj9Hems3HmEH7yxhspqKV5CBOPyvu159e5hbCw5weT5q6moitycko0kDWREc8Yjo/o2q0MwmN1VfTH73tnNGWm4PbU89u4m7n9zHbPGDMNpl7GWEM01elBHXrwjnZ+8vZ5pi4qZMy4Tl58VfKxK/goYxKhpNaOvnoK5R2XGumsN3Z3VlaeuH8DyLQfqOi1rI259ZiEs5cahqTx/6xD+s/0w9y1Zi6em1uyQmkyuuAzib1qtqUXHqKunxlavb4yvq79wd31NvKgHbk8tz3+0jXiHjedvHYLN5mv5YCFEY+7I7EKlp4Yn39vCT95az8t3DcURQbMZUrgMYva0mjfBFtOz32OFPZB++N1euD01vPyP7bicdp6+cSDKy8aZQojA5I7sTmV1Lc98uJV4h40Xbk+PmAGhFC4/mnLPytdeWGYuAmpEMQ3FvbPm+slVfXB7apj9yU5cThuPf7+/FC8hgjDlkp5UVNXw+4+/IN5p59mbB0VETknh8qGp02xWmVarL1zFNFw70yqlePSafrg9Ncz5zy4SnHYe+p48rCpEMO67ojcVnhr++K8viXfYeOr6AZYvXlK4fGjqNFsop9WaWxjCUUyDvY/WVEopnrp+IG5PLa+s2EG80869l/c2/DxCxAqlFI+M6ovbU8vcT3eREGfnZ6P6Wrp4SeHyoTnTbKGYVgumMITjHpWRTSmBstkUz94yGHd13XlcTjuTL+4RknMJEQuUUjx5XX/c1TW89q8vSXDauf/KPmaH5ZMULh+scs/KiAaLUN6jMqspxW5T/P72dKqqa/n1B5/hctoYm90tpOcUIpoppXjmxkG4PTW8+PEXuJw2pl3ay+ywvIqc/scws8oCoFbsVqzPzGe9HHYbL9+VwRX92vOLZZt5Z01JyM8pjBepe8lFI5tN8btbh3DtkE48+5dtLFz5ldkheRVU4VJKzVRKbVNKbVRK/VkplWxUYGazygKgVngI2B+zC3ycw8Yfxw7jol4pPPI/G/hg476wnDdUojmnvDFrPUzhm8Nu46U7h3L1gA788r0tLF291+yQzhPsFdfHwCCt9RDgC+Cx4EOyDissAGp2YWiMFQq8y2knb9xwhne7gJ+8tZ6/bfk6bOcOgajOqYbMXA9T+Oa025g1JoNL+qTw83c38t56aw0kgrrHpbX+W71PVwG3BReOaMhKDwH7YoVnvRLjHMydMIJ7Coq4b8k65ozP5LIL25kaU3PEWk5ZfSo8lsU77OTlZjJxfhEPLd1AvMPG6EGdzA4LMPYe1yTgr76+qJSappQqVkoVHzp0yMDTho9Zc/FWuPKLBC1dThZOzKJ3+ySmLSxm5ZdHzA4pWFGfU1afCo91CXF28sePID2tNT9+cx3/3HbQ7JCAAAqXUurvSqnNXj5urPc9TwDVwGJfx9Fa52mtM7XWme3aRd5IWObiI0PrRCeLJmfRtU0ikxesZs3uY2aHdB7JqW9ZfSpcQFK8g3kTs+jbsSXT31jDpzsOmx1S44VLa32V1nqQl4/3AJRS44HrgLFa+9hvPgrIXHzkaJsUz+Ip2bRvGc+EuUVsKjlhdkjnkJz6lhXukYrGtU5wsmhSNj3atmDKgmJWf3XU1HiC7SocDfwcuEFrXW5MSNYkc/GRpX0rF4un5tAqwUnu3EK2fX3S7JACEks5dZZMhUeGC1rE8caUbDolu5g4bzXr9x43LZZg73HNAloCHyul1iulXjcgJkuSufjIk5qcwJtTc4h32Lgnv5AvD502O6RAxExOicjTrmU8S6bk0KZFHOMKCtmyz5zZjKAKl9a6t9a6i9Z66JmPHxgVmNXIXHxk6to2kcVTcgAYO6eQPUesfRETSzklIlPH1i4WT8kmKd5BbkER2w+cCnsMsnJGgGQuPnL1bp/EG1OycVfXcPecVTK9K0SQurRJZPHUHOw2xZj8QnYdLgvr+ZUZ934zMzN1cXFx2M8rYtumkhOMmbOKtklxLJ0+kvatXM0+llJqjdY608DwgiI5Jcyw/cAp7sxbhcth4+3pI+nSJjGo4wWaV3LFJWLG4LTWzJ80goOnKhmbX8iR05VmhyREROvToSWLJmdxurKasfmFfH3CHZbzSuESMWV4tzYUjB/BnqPl5BYUcaLcY3ZIQkS0gZ1bs3ByNkfLqhiTv4pDp0I/IJTCJWLOyF5tyRuXyY6Dpxk3r4hTbileQgRjaJdk5k4Ywf7jbnILCjlWVhXS80nhEjHpsgvb8Yexw9hSeoLJ84spr6o2OyQhIlpWjzbMGZfJzsNljJtbxMkQDgilcImYdfWADrx011CKdx9l2sI1uBusjCKEaJqL+6Tw+j3D2Pb1SSbOW01ZZWgGhFK4REy7bkhnZt6Wzn93HOZHi9dSVV1rdkhCRLQr+nXglbsyWL/3OJMXrA7JgFAKl4h5tw5P4zc3D2LFtoM88NY6qmukeAkRjGsGd+L3t6dTuOso0xatobLa2OIlhUsIYGx2N568bgB/3fw1D/9pAzW1Ub22rRAhd1NGKs/dMphPvjjEfUvW4TFwQCiFS4gzJl/cg0dG9eW99ft44s+bqJXiJURQ7hzRlV/dMJCPPzvAg2+vN2xAKIVLiHruvbw3P76iN28X72XtHuvt5SVEpBn/ne48dk0/Pti4n/9sN2bDU4chRxEiijx09YVcPaADQ9KSzQ5FiKgw/bJejOzV1rCckisuIRpQSknREsJgRuaUFC4hhBARRQqXEEKIiBJU4VJK/VoptfHMTq1/U0p1NiowIWKR5JQQjQv2imum1nqI1noo8AHwSwNiEiKWSU4J0YigCpfW+mS9T1sA8uCLEEGQnBKicUG3wyulfgOMA04Al/v5vmnAtDOfnlZKfR7suQOUAhwO07kaY6VYQOLxp7FYuoXqxJJTTWKlWEDi8SeQWALKK6W1/wGdUurvQEcvX3pCa/1eve97DHBprZ8K5MThopQqtsoW61aKBSQef0IZi+SUcawUC0g8/hgZS6NXXFrrqwI81hLgQ8BSSSaE1UhOCRGcYLsK+9T79AZgW3DhCBHbJKeEaFyw97ieU0r1BWqB3cAPgg/JcHlmB1CPlWIBiccfs2KRnGoaK8UCEo8/hsXS6D0uIYQQwkpk5QwhhBARRQqXEEKIiBL1hUspNVMpte3MMjp/VkqZuuy3Uup2pdQWpVStUsq0NlWl1Gil1OdKqR1KqUfNiuNMLHOVUgeVUpvNjONMLF2UUv9USm098//pAbNjshrJKZ9xSE55j8XwnIr6wgV8DAzSWg8BvgAeMzmezcAtwCdmBaCUsgN/AK4BBgB3K6UGmBUPMB8YbeL566sGHtZa9wdygHtN/t1YkeRUA5JTfhmeU1FfuLTWf9NaV5/5dBWQZnI8W7XW4VrhwJcsYIfWeqfWugp4C7jRrGC01p8AR806f31a6/1a67Vn/vsUsBVINTcqa5Gc8kpyyodQ5FTUF64GJgF/NTsIC0gF9tb7vAT543wepVR3IAMoNDcSS5OcqiM5FQCjcirotQqtIJAldJRST1B3ybrYCvGYTHl5TZ6LqEcplQS8A/ykwcK3MUFyqskkpxphZE5FReFqbAkdpdR44DrgSh2GB9easKSPWUqALvU+TwP2mRSL5SilnNQl2GKt9btmx2MGyakmk5zyw+icivqpQqXUaODnwA1a63Kz47GI1UAfpVQPpVQccBfwvskxWYJSSgEFwFat9Ytmx2NFklNeSU75EIqcivrCBcwCWgIfn9lV9nUzg1FK3ayUKgFGAh8qpZaHO4YzN9bvA5ZTd6N0qdZ6S7jjOEsp9SawEuirlCpRSk02KxbgIiAXuOLM+2W9Uur7JsZjRZJTDUhO+WV4TsmST0IIISJKLFxxCSGEiCJSuIQQQkQUKVxCCCEiihQuIYQQEUUKlxBCiIgihUsIIUREkcIlhBAiovx/w0pKTU3h2b8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "##########################\n",
    "### 2D Decision Boundary\n",
    "##########################\n",
    "\n",
    "w, b = model1.weights.view(-1), model1.bias\n",
    "\n",
    "x_min = -2\n",
    "y_min = ( (-(w[0] * x_min) - b[0]) \n",
    "          / w[1] )\n",
    "\n",
    "x_max = 2\n",
    "y_max = ( (-(w[0] * x_max) - b[0]) \n",
    "          / w[1] )\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3))\n",
    "\n",
    "ax[0].plot([x_min, x_max], [y_min, y_max])\n",
    "ax[1].plot([x_min, x_max], [y_min, y_max])\n",
    "\n",
    "ax[0].scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')\n",
    "ax[0].scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')\n",
    "\n",
    "ax[1].scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], label='class 0', marker='o')\n",
    "ax[1].scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], label='class 1', marker='s')\n",
    "\n",
    "ax[1].legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## High-level implementation using the nn.Module API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogisticRegression2(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, num_features):\n",
    "        super(LogisticRegression2, self).__init__()\n",
    "        self.linear = torch.nn.Linear(num_features, 1)\n",
    "        # initialize weights to zeros here,\n",
    "        # since we used zero weights in the\n",
    "        # manual approach\n",
    "        \n",
    "        self.linear.weight.detach().zero_()\n",
    "        self.linear.bias.detach().zero_()\n",
    "        # Note: the trailing underscore\n",
    "        # means \"in-place operation\" in the context\n",
    "        # of PyTorch\n",
    "        \n",
    "    def forward(self, x):\n",
    "        logits = self.linear(x)\n",
    "        probas = torch.sigmoid(logits)\n",
    "        return probas\n",
    "\n",
    "model2 = LogisticRegression2(num_features=2).to(device)\n",
    "optimizer = torch.optim.SGD(model2.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | Train ACC: 0.973 | Cost: 0.055\n",
      "Epoch: 002 | Train ACC: 0.973 | Cost: 0.053\n",
      "Epoch: 003 | Train ACC: 0.973 | Cost: 0.051\n",
      "Epoch: 004 | Train ACC: 0.973 | Cost: 0.049\n",
      "Epoch: 005 | Train ACC: 0.973 | Cost: 0.048\n",
      "Epoch: 006 | Train ACC: 0.973 | Cost: 0.047\n",
      "Epoch: 007 | Train ACC: 0.973 | Cost: 0.046\n",
      "Epoch: 008 | Train ACC: 0.973 | Cost: 0.045\n",
      "Epoch: 009 | Train ACC: 0.973 | Cost: 0.044\n",
      "Epoch: 010 | Train ACC: 0.987 | Cost: 0.043\n",
      "Epoch: 011 | Train ACC: 0.987 | Cost: 0.042\n",
      "Epoch: 012 | Train ACC: 0.987 | Cost: 0.041\n",
      "Epoch: 013 | Train ACC: 0.987 | Cost: 0.041\n",
      "Epoch: 014 | Train ACC: 0.987 | Cost: 0.040\n",
      "Epoch: 015 | Train ACC: 0.987 | Cost: 0.039\n",
      "Epoch: 016 | Train ACC: 0.987 | Cost: 0.039\n",
      "Epoch: 017 | Train ACC: 1.000 | Cost: 0.038\n",
      "Epoch: 018 | Train ACC: 1.000 | Cost: 0.038\n",
      "Epoch: 019 | Train ACC: 1.000 | Cost: 0.037\n",
      "Epoch: 020 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 021 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 022 | Train ACC: 1.000 | Cost: 0.036\n",
      "Epoch: 023 | Train ACC: 1.000 | Cost: 0.035\n",
      "Epoch: 024 | Train ACC: 1.000 | Cost: 0.035\n",
      "Epoch: 025 | Train ACC: 1.000 | Cost: 0.034\n",
      "Epoch: 026 | Train ACC: 1.000 | Cost: 0.034\n",
      "Epoch: 027 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 028 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 029 | Train ACC: 1.000 | Cost: 0.033\n",
      "Epoch: 030 | Train ACC: 1.000 | Cost: 0.032\n",
      "\n",
      "Model parameters:\n",
      "  Weights: Parameter containing:\n",
      "tensor([[5.0453, 3.4349]], requires_grad=True)\n",
      "  Bias: Parameter containing:\n",
      "tensor([-0.7931], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "def comp_accuracy(label_var, pred_probas):\n",
    "    pred_labels = custom_where((pred_probas > 0.5).float(), 1, 0).view(-1)\n",
    "    acc = torch.sum(pred_labels == label_var.view(-1)).float() / label_var.size(0)\n",
    "    return acc\n",
    "\n",
    "\n",
    "num_epochs = 30\n",
    "\n",
    "X_train_tensor = torch.tensor(X_train, dtype=torch.float32, device=device)\n",
    "y_train_tensor = torch.tensor(y_train, dtype=torch.float32, device=device).view(-1, 1)\n",
    "\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    #### Compute outputs ####\n",
    "    out = model2(X_train_tensor)\n",
    "    \n",
    "    #### Compute gradients ####\n",
    "    cost = F.binary_cross_entropy(out, y_train_tensor, reduction='sum')\n",
    "    optimizer.zero_grad()\n",
    "    cost.backward()\n",
    "    \n",
    "    #### Update weights ####  \n",
    "    optimizer.step()\n",
    "    \n",
    "    #### Logging ####      \n",
    "    pred_probas = model2(X_train_tensor)\n",
    "    acc = comp_accuracy(y_train_tensor, pred_probas)\n",
    "    print('Epoch: %03d' % (epoch + 1), end=\"\")\n",
    "    print(' | Train ACC: %.3f' % acc, end=\"\")\n",
    "    print(' | Cost: %.3f' % F.binary_cross_entropy(pred_probas, y_train_tensor))\n",
    "\n",
    "\n",
    "    \n",
    "print('\\nModel parameters:')\n",
    "print('  Weights: %s' % model2.linear.weight)\n",
    "print('  Bias: %s' % model2.linear.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluating the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 96.00%\n"
     ]
    }
   ],
   "source": [
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32, device=device)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.float32, device=device)\n",
    "\n",
    "pred_probas = model2(X_test_tensor)\n",
    "test_acc = comp_accuracy(y_test_tensor, pred_probas)\n",
    "\n",
    "print('Test set accuracy: %.2f%%' % (test_acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAADFCAYAAAAMsRa3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX5+PHPmSWZhAARwpqwg+yEQEhC3epWsO67gmFf2mq1am1daqXWWi3WnwutEhJWQaVfLfrVVmpLW1u/kBB2EBQEgQRkX5NMMknO74+AhjAzmWTuzL0z87xfr7xeZpLc+xDnyXPuuc89R2mtEUIIISKFzewAhBBCiKaQwiWEECKiSOESQggRUaRwCSGEiChSuIQQQkQUKVxCCCEiihQuIYQQEUUKlxBCiIgihUsIIUREcZhx0pSUFN29e3czTi2EIdasWXNYa93O7DjOkpwS0SDQvDKlcHXv3p3i4mIzTi2EIZRSu82OoT7JKRENAs0rmSoUQggRUaRwCSGEiChSuIQQQkQUU+5xCSFCz+PxUFJSgtvtNjsUS3G5XKSlpeF0Os0ORTRT1BauyuoaHn1nExO+0530LslmhyNE2JWUlNCyZUu6d++OUiro49Vqzb5jFSS3iCMpPjL/dGitOXLkCCUlJfTo0cPscEQzRe1U4YlyD6u/Osq4uUVs3X/S7HCECDu3203btm0NKVoAtbWasqoavjpcRnlltSHHDDelFG3btpWr0AgXdOFSSrmUUkVKqQ1KqS1KqV8ZEViw2rdysWRKDglOO/fkF7Lj4GmzQxIiIEbmlFFFC8Bht9EzpQUOu2LXkTIqqiK3eInIZsQVVyVwhdY6HRgKjFZK5Rhw3KB1bZvI4qnZKKUYm7+K3UfKzA5JiEBYNqecjrriZVOKXYfLcHtqzA5JxKCgC5euc/ZyxnnmQwd7XKP0apfE4inZVFXXMmZOIaXHK8wOSQi/rJ5TcQ47PVNaoJRi56EyKptYvGbMmMELL7wQktjWrFnD4MGD6d27N/fffz9aW+bXJgxkyD0upZRdKbUeOAh8rLUu9PI905RSxUqp4kOHDhlx2oD17diSRZOzOen2MGbOKg6clPltYW1m5NSydaVc9NwKejz6IRc9t4Jl60p9fm+8006PlBYA7DxcRlW1Na68fvjDH5KXl8f27dvZvn07H330kdkhiRAwpHBprWu01kOBNCBLKTXIy/fkaa0ztdaZ7dqFf4m3QamtWTApi8OnKhkzZxWHT1eGPQYhAhXunFq2rpTH3t1E6fEKNFB6vILH3t3kt3i5zhSvWq3PFK/a875n4cKFDBkyhPT0dHJzc8/7+pw5cxgxYgTp6enceuutlJeXA/CnP/2JQYMGkZ6ezqWXXgrAli1byMrKYujQoQwZMoTt27efc6z9+/dz8uRJRo4ciVKKcePGsWzZsiB+K8KqDO0q1FofB/4FjDbyuEYZ1vUCCiaMoPR4BffkF3K8vMrskITwK1w5NXP551Q0mPKr8NQwc/nnfn8uIa6ueNXUaHYdLsNT823x2rJlC7/5zW9YsWIFGzZs4OWXXz7v52+55RZWr17Nhg0b6N+/PwUFBQA8/fTTLF++nA0bNvD+++8D8Prrr/PAAw+wfv16iouLSUtLO+dYpaWl57yWlpZGaanvwisilxFdhe2UUsln/jsBuArYFuxxQyWnZ1vycjPZeaiMcXOLOOn2mB2SEOcwI6f2+bj36+v1+hLjHHRPaYGnppZdh8uoPlO8VqxYwW233UZKSgoAbdq0Oe9nN2/ezCWXXMLgwYNZvHgxW7ZsAeCiiy5iwoQJzJkzh5qauoI6cuRInn32WZ5//nl2795NQkLCOcfydj9LOgijkxFXXJ2AfyqlNgKrqZuP/8CA44bMpRe2449jh/HZvpNMmrea8ght6xVRK+w51Tk5oUmvN9Qi3kH3tolUVp8pXrW1aK0bLRwTJkxg1qxZbNq0iaeeeuqb56tef/11nnnmGfbu3cvQoUM5cuQIY8aM4f333ychIYFRo0axYsWKc46VlpZGSUnJN5+XlJTQuXPngOIXkcWIrsKNWusMrfUQrfUgrfXTRgQWalcN6MDLd2Wwds8xpiwolrZeYRlm5NQjo/qS4LSf81qC084jo/oGfIwkl5NubRJxV9fy1eFyvnv5FSxdupQjR44AcPTo0fN+5tSpU3Tq1AmPx8PixYu/ef3LL78kOzubp59+mpSUFPbu3cvOnTvp2bMn999/PzfccAMbN24851idOnWiZcuWrFq1Cq01Cxcu5MYbb2zKr0FEiKhdOSMQ1w7pxAu3p7Ny5xF+8MYaKi3SGSVEuN2UkcpvbxlManICCkhNTuC3twzmpozUJh2nVYKTrm0SqKiqoUXH7jz22ONcdtllpKen89BDD533/b/+9a/Jzs7m6quvpl+/ft+8/sgjjzB48GAGDRrEpZdeSnp6Om+//TaDBg1i6NChbNu2jXHjxp13vNdee40pU6bQu3dvevXqxTXXXNPk34WwPmXGcw6ZmZnaSpveLSncw+N/3sSogR2YNWYYTntM13MRAKXUGq11ptlxnOUtp7Zu3Ur//v1Nied4eRV7jpaTFO+ge9sW2GzWutdk5u9G+BZoXslfaGBMdld+ed0Alm85wMNLN1BTKw8tChGM5MQ40i5I5HRlNXuOllMrDwILA0XmEs8hMOniHrira/jdR58T77Dx/K1DLDdKFCKStGkRV7ei/PEK9h4tp2ubROnyE4aQwlXPj77bG7enllf+sR2X087TNw6URBMiCClJ8WgN+09UUHKsgrQLEiSnRNCkcDXw4FV9cHtqyPtkJy6njce/318STYggtGsZT63WHDjpRqm6xg/JKREMKVwNKKV47Jp+uD01zPnPLhKcdh76XuAtwUKI87VvGY/WmoOnKrEpRafWLileotmkcHmhlGLG9QNxe2p4ZcUO4p127r28t9lhCRGxlFJ0aOWiVsPh05UoBR1bSfESzSNdhT7YbIrf3jKEG4d2Zubyzyn47y6zQxIioqkzV1rzX53JCzNf4OAp4xe6fuKJJ+jSpQtJSUmGH1tYh1xx+WG3KX5/ezqVnlp+/cFnuJw2xmZ3MzssIYz3bCpUedklPC4JHjduoVqlFC1dDmrsdg6cdGNT0K6ly7DjX3/99dx333306dPHsGMK65ErrkY47DZeuTuDy/u24xfLNvPOmpLGf0iISOOtaPl7PUDetjVRStE6wUnrBCezXptNxvBMQ7Y1AcjJyaFTp05BxSysT664AhDnsPHaPcOZvGA1j/zPBuKdNq4bIot3CuHP2W1NPv30U1JSUs5Zq1ApRZc2idx0083cOmY8aRck8uJvn6agoIAf//jH32xrkpqayvHjx4FvtzUZO3YsVVVV36waL2KPXHEFyOW0M2dcJsO7XcBP3lrP37Z8bXZIQlhaY9ua2JTieOlOJt/2fbKHD2XRG28Eta2JiB1SuJogMc7B3AkjGNi5FfctWce/vwh+u3QholUg25pMmjSRvNf+yEf/KWLq/T/jZFndVGFztjURsUMKVxO1dDlZMCmLXu2TmLawmJVfHjE7JCEs6corrwxoW5PU1M6ktorjr+/9D2WV1Zys8DRrWxMRO6RwNUNyYhxvTM6iS5tEJi9YzZrdx8wOSYjgxPloH/f1egAGDhzIE088EdC2JqNHfY+hgwdgV4rdR8t58KGHm7Wtyc9+9jPS0tIoLy8nLS2NGTNmNDt+YV2yrUkQDp50c8fslRw5XcWSqTkMTmttdkgiTGRbk9Corqll5+Eyqqpr6ZHSghbxoekfi8TfTSwI27YmSqkuSql/KqW2KqW2KKUeCPaYkaJ9KxeLp+bQKsFJ7txCtn190uyQRBSI5Zxy2G30SGmB027jq8NllFdVmx2SsCAjpgqrgYe11v2BHOBepdQAA44bEVKTE1gyNZt4h4178gv58lBwz70IQYznlPNM8bLbFbsOl1EhxUs0EHTh0lrv11qvPfPfp4CtQNP2+45w3dq2YPGUHADGzilkz5FykyMSkczInDLjVoAR4hw2eqa0wKYUuw6X4/YY98xWpP5OxLcMbc5QSnUHMoBCL1+bppQqVkoVHzoUfW3kvdsnsWhyNu7qGu6es4p9xyvMDklEgWByyuVyceTIkYj9Qx3nsNMzpQUo2Hm4jEoDipfWmiNHjuByGbfMlAg/w5ozlFJJwL+B32it3/X3vdHSnOHNppITjJmzirZJcSydPpL2rSRBolE4mjOCzSmPx0NJSQlutzuEUYaep6aWw6cqUUqR0jIOhy248bbL5SItLQ2n02lQhMIogeaVIS07Sikn8A6wuLEEi3aD01ozf9IIcguKGJtfyFvTcmibFG92WCLCGJFTTqeTHj16GBuYSTaX1g0IkxPrBoQdW8uAMJYZ0VWogAJgq9b6xeBDinzDu7Uhf3wme46Wk1tQxIlyj9khiQgiOXW+QamtWTApiyOnKxmTv4pDIdgSRUQOI+5xXQTkAlcopdaf+fi+AceNaN/plcLs3OHsOHiacfOKOOWW4iUCJjnlRUbXC5g7YQT7jleQW1DIsbIqs0MSJjGiq/C/WmultR6itR565uMvRgQX6b7btz2zxmSwufQEk+cXyzMpIiCSU75l92xL/rgR7Dxcxri5RZyUAWFMkm1NQux7Azvy0p1DeeCtdUxbuIb88Zm4nHazw2pcoBsLhmkDQiHOurhPCq+NHcb0RWuYOG81CydlhWyFDWFN8n87DK5P70xldS0//dMGfrR4La/fM5w4h8WXiQx0Y8EQbUAohD9X9u/Aq3dncO+StUxesJr5E7MiY0DYkK+Bny8yIARkkd2wuW14Gs/cNIgV2w7ywFvrqK6pNTskISLaNYM78eIdQyncdZTpi9ZQWR2BG0s2dYAnA0JArrjC6p6cblRW1/LrDz7jp3/awO/vGIrd5n+/IiGEbzdlpFJZXcPP39nEfUvW8cexw3Dao3w8PsPLYt4xdiUW5f+HrWfyxT14ZFRflq3fxxN/3kRtbWSuaiCEVdw5oiu/umEgH392gAffXk9NLOZUjF2JyRWXCe69vDduTw2vrthBvMPGjBsGNrpTrKU0dV5eiBAb/53uuD01/Pav24h32Jl52xBsMpsRtaRwmeShqy+koqqG/P/uwuW08+g1/axVvOKSfHcLBlK0gtiA0DDS8RhTpl/WiwpPDS/9fTsup41nbhpkrZwShpHCZRKlFE9c2x93dQ2zP9mJy2nnwasvNDusb/n7w+5tjv2br51o/NjhKij+Oh7r/xukkEWNB67sQ4Wnhtn/rsupX1zb39rFK9CBoDiHFC4TKaV4+oZBuD21vPyP7bicdn743V5mhxV6Vmuhlz8cUUMpxaOj+1HpqaXgv7tIcNr56ai+ZoflW1MHTP4GjTFECpfJbDbF87cOobK6luc/2kaC08aEi6JjYVQhzKCU4pfXDcDtqWHWP3fgctq474o+ZodlDH9T+DFECpcF2G2KF+9Ip9JTw4z//Yx4p527s7qaHda5pCFDRBCbTfGbmwdTWV3LC3/7ApfTzpRLepodVvBkShuQwmUZTruNV8dkMG3hGh7/8yZcThs3Z6SFP5BgClSMjfqEtdltipm3DaGyuoZnPtxKvNNObk43s8MSBpDCZSHxDjuzc4czcd5qHl66gXiHne8P7hTeIJpTtAJpyAj4WA3m8JvbOPFsE3e6P3teadSIKg67jZfuzKDSs4Ynl23G5bBxe2YXs8MSQZLCZTEup5388ZmMm1vE/W+uI95h48r+HcwOK3CBdAw2pZOqfgdgU4pKc68aZTo06sQ5bPxh7DCmLCjm5+9sJN5p54b0zuENQh7NMJSsnGFBLeIdzJs4ggGdW/HDN9byn+2HzA4pcIF0DD5eWneVVv8jmGM3lZFXiCIiuJx28sYNJ7NbGx58ez3Lt3wd3gCs1kkb4eSKy6JauZwsnJTFXXmrmLqwmAUTs8ju2dbssLwLtEXX36hTiBBLjHMwd+II7skv5L4la5kzLpPv9m1vdliiGQy54lJKzVVKHVRKbTbieKJOcmIcb0zJJjU5gUnzV7NuzzGzQwqOjDoDJjkVGknxDhZMzKJP+5ZMX7SG//vysNkh1Q3oZrQ+/6Op92ljiFFThfOB0QYdS9STkhTPkqk5pLSMZ/zcIjaXhniaS65+rGI+klMh0TrRyRtTsunaJpEpC4op/uqo8SdpWIz8kQFdkxlSuLTWnwAh+L8vADq0crF4SjYtXU5yCwr54sCp0J2s4f2ncDG6YPo63tnXG/u6ySSnQqtNizgWT82mQysXE+etZmPJcWNPIEUnpJTWxmwBoJTqDnygtR7k4+vTgGkAXbt2Hb57925DzhtLvjpcxh2zV1KrYen0HHq2C8Ef2VA9aNxYJ+GME81fzsaEziyl1BqtdWaIz9EdyamQ2ne8gjtmr+SUu5q3puXQv1MrYw4c6Hs5kLyIIYHmVdi6CrXWeVrrTK11Zrt27cJ12qjSPaUFS6Zmo7VmbH4he4+WG38SI4pWw47BGSdCW1hidHQrORW8zskJvDk1hwSnnXvyC9lxMAzvpXDlRRSTdvgI07t9SxZNzqa8qoYx+avYf6LC7JDO5W+qzeLTcyI2dWmTyJKp2SilGJu/it1HyswOKbSioBlE2uEj0IDOrVg4KYux+YWMnVPIW9NzaN/SFb4AzhYab1c6/h4YltGlsKie7ZJYPCWbu/JWMmZOIUt/MJLU5ITwnDzcC+dGQTOIUe3wbwIrgb5KqRKl1GQjjit8S++SzLyJI9h/wk1ufhFHy6rCd/Kq04G9ySMoEaxGcir8+nasm8046fYwZs4qDpx0N/9gTZld8PZAvkwj+mXIFZfW+m4jjiOaZkT3NhSMz2TC/NXkFhSyZGoOrROcZofVdLLy/Hkkp8wxKLU1CyZlkZtfyNj8Qt6alkNKUnzTDyRFJ6TkHleE+07vFGbnDueLA6eYMK+I05XVwR3QjPtNwRYtuUcmDDSs6wUUTBhBybFycguKOF4extkMERApXFHg8r7tefXuYWwsOcGk+aupqKpp/sHqT1tYmUypiBDK6dmWvNxMvjx4mvFzizjp9pgdkqhHCleUGD2oIy/ekc7qr44ybVExbk8zildTnvYXIspdemE7/jh2GFv2nWTSvNWUVwU5m2EVUdDdK12FUeTGoams+vIIb67eS78nP6Jzaxc/G92PmzICbHM1+j5TBCWCEN5cNaADL9+VwY/fXMuUBcXMnTACl9NudljBiYIZCilcUWTZulKWrd/3zef7Trh59J2NAIEXr2BZfYpRiCa6dkgnKqvTefhPG/jBG2uYnTuceEeEF68IJ4Urisxc/jkVDaYI3dW1/O6jbcYWLqOfO2nKxpLBnEeIZrplWBpuTy2P/3kT97+5jj+MGYbDbtCdlqZsMikbUgJSuKLKvuPeV9HYd8KN1hqllDEn8pYgZxOq4b2xQBLq8dKmtcTHUIIK6xiT3RW3p4anP/iMh5Zu4P/dORS7zYCcasoDwVHw8LARpDkjinT286T/r/73M4xaUNmrYBPKrFXphWiCSRf34Gej+/L+hn08+s5GamtDmFMQUcswhVPUX3EtW1fKzOWfs+94BZ2TE3hkVN/w3e8Js0dG9eWxdzedM13octjI7tmW+f/3FQlxdn42qq/vK69Apuxkmk7EuB99tzduTy2v/GM7Lqedp28caNxsRkMxdiUVqKguXMvWlZ7zh7z0eAWPvbsJCGOzQhid/Tc1LNQ3Du3ME8s289q/viTBaef+K/t4P0C4p+D8zdf7U386Msbm9oU1PHhVH9yeGvI+2YnLaePx7/f/pnjF0mDZLFFduLw1K1R4api5/POofSPdlJHq9d/2zI2DcHtqePHjL3A5bUy7tFf4gvL2TJi/q7uq04E3bMiIVJhAKcVj1/TD7alhzn92keC089D3+po7WI6hxo2oLlw+mxV8vB5u4RyZ2WyK3906hMrqWp79yzZcTjvjRnYPybkC0ljBaZho8kC0sBilFDOuH0ilp5ZXVuwg3mlnSeGepg+Wm9JV66+jN4YaN6K6cHVOTqDUS5Hy18QQLmaMzBx2Gy/dOZSq6lp++d4WXA47d4zoYszBm9rSLkQUsNkUz94yGHd1XXHyxe9g+fHSwAdm/q6cQj24s9AVXVR3FT4yqi8JDZ5yT3DaeWRUX5Mi+pa/acxQctptzBqTwSV9Uvj5uxt5b71Bb7jHS6VxQ8Qku03x+9vTGT2wo8/vaXSwHAnLMFnoii6qr7h8NStY4f6WmdOY8Q47ebmZTJxfxENLNxDvsDF6UKfgDyxXXCJGOew2Xrk7g5v+8Cmf7T95ztcCGixH2T2oUIvqwgW+mxXMZvY0ZkKcnfzxIxhXUMiP31xHXq6dy/u1P/8bQzk94G96seEuyuHeJVaIJopz2Hj3R9/hhln/5YsDde/VVAsNlqOJIYVLKTUaeBmwA/la6+eMOG408/bMVbinMZPiHcybmMW1r/yHifNXA14SLVTTAw0Ln6/5+bPnibERqeRU8MxoS3c57Sy79yLGzy1i3Z7jzLhhIFcP6BDSc34jhgZ3QRcupZQd+ANwNVACrFZKva+1/izYY0czq0xj/nPbQQ6fqvzm85A1ichqGAGTnAqemW3piXEO5k4YwT35hdy7eC1zxmdy2YXtQnpOIKYGd0Y0Z2QBO7TWO7XWVcBbwI0GHDfq3ZSRyqePXsGu567l00evMGU6Yebyz3FX157zWjiaRIRfklNBMqv56ayWLicLJmXRq30S0xYWs/LLI2E5b0hZqIHEiKnCVGBvvc9LgOyG36SUmgZMA+jatasBpxVG8NUM4u3+W6NiaKoixCSngmSFZziTE+N4Y3IWd+atYvKC1SyanM3wbheE7fyGs9AVnRGFy9siXeetPKm1zgPyADIzM0O8MqX5rLbsi694fDWJ2BRsP3AKH4tDeWehN3aEk5wKktnNT2e1TYpnyZRs7pi9kglzi1gyNYfBafIwfbCMmCosAeo/xZoG7PPxvTHh7Px66fEKNN/Ory9bZ84fdn/xeHvWLd5hIynewZj8QnbZe3o/qNFXURaahrAAyakgWekZzvatXCyemkOrBCe5cwv5/OtTYY8h2qhgt7pQSjmAL4ArgVJgNTBGa73F189kZmbq4uLioM4bDs29arrouRVeR3upyQl8+ugVoQg1qHi8/TsHdm7FnXmrcDlsvD19JF3aJJ77wxZ6it4MSqk1WuvMEB07anMqnKw267H7SBl3zF5JTa3m7ekj6dUuJgdlfgWaV0FPFWqtq5VS9wHLqWvdnesvwSJFMF1JzZ1fD1WiNRaPr2fdFk3O4u68VYzNL2Tp9JF0bO369osWeoo+2kRrToWb1Z7h7Na2BYun5HBX3krGzqnLqa5tExv/QXEeQ5Z80lr/RWt9oda6l9b6N0Yc02zBdCX5mkf3N78eyunF5sQDMLBzaxZOzuZoWRVj8ldxqF7bvAitaMwpAb3bJ7Focjbu6hrG5K+yzILfkSaq1ypsrmXrSn121QXyRmvO/Hoo23eDme8f2iWZuRNGsP+4m9yCQo6VVQUdjxCxrH+nViyalM2Jcg9j5qzi4Em32SFFHClcDZy98vElkK6kmzJS+e0tg0lNTkBRdy/pt7cM9jttEcr23ebEU19WjzbMGZfJzsNljJtbxIkKT9AxCRHLBqe1Zv6kERw8VcnY/EKOnJbZjKaI+rUKm8rblc9ZTelKaur8eqjbd4Od77+4Twqv3zOM6YvWMHFeEYt0PC2UJJvwz2oNElYyvFsbCsaPYMK8InILinhzag6tE51mhxURLHvFtWxdKRc9t4Iej37IRc+tCFsrub8rnKZcpTSVt+k8Rd29rnD++/25ol8HXrkrgw0lJ5hc/Shu7SXJYrN9XXhhtcdCrGhkr7bkjctkx8HTjJtXxCm3zGYEwpKFy8w3vK8rnNTkhJCOFOtP50Fd0Tr7oIKVEv6awZ34/e3pFNb2ZVq3v1D5i6N16xCe/YiBVngRGLOXXYoUl13YjlljMthceoLJ84spr6o2OyTLs2ThMvMNb+aDi2fXLkxNTjhvmQQrJfxNGak8d8tgPvniEPctWYenprbxHxIxxwrLLkWK7w3syEt3DqV491GmLVyD28ftClHHkoUrFG/4QKceg21kMEIkJPydI7ryqxsG8vFnB3jw7fXU1MqKQ+JczX0MI1Zdn96Z392Wzn93HOZHi9dSVS0DQl8s2ZxhdKNCUx8mDseDi/VvWrdOcKIUHC/30Dk5geREJ8fKz5/rtlrCj/9Od9yeGn77123EO+zMvG0INpu3ZfZELArXnnOR0gASSJy3DU/D7anhF8s288Bb63j17gwcdkteX5jKkr8Ro6frjJp6NKphpOE9vOMVHo6Ve765n3es3IO9QQEwa521xky/rBcPXnUh76wt4cn3NhPsEmIieoRj9iJSGkCaEuc9Od148roB/HXz1/z0TxtkNsMLS15xGb3JohFTb0ZuTOev5f6smlrNBWeuvOxKnVNorTaavP/K3lR4anj9318S77Dz5HX9UUquvEToZy98DUofXrqBB99eb5krMH+DZ2+xTb64B+4zX3c57Tx782CZzajHkoULjH3DGzH12NQ3nj+BFkyt6660zNjFtSmUUvx8dF/cnhrmfrqLxDg7P7Xg1aGIPr5yqebMlb9VcqY5g+d7L++N21PDqyt24HLaeer6ATIgPMOSU4VGM2LqsTlvPF9Ti4EWzOMVnohpJ1ZK8dT1A7g7qwuz/rmDWSu2mx2SiAGB5JIVcqa5jSoPXX0hUy7uwfz/+4rnPtomU/FnWPaKy0hGTD029arN39Sit5vWTVG/WFrpxrRSimduGozbU8sLf/sCl9POlEt87OclhAECzaVAZzlClU/NbVRRSvHEtf1xV9cw+987SXDa+clVFwYdT6SLicIFwU89Xt6vHYtX7Tnn+Sp/bzx/U4tn9+Q6myAJThvlnnNbXxOcdlxOm9/uQiPvuxnFblPMvG0IldU1PPPhVuKddnJzupkSi4h+DQelNqW+mSasL5Ars1DmUzCDZ6UUT98wCLenlpf+vh2X084PLusVVDyRLmYKVzCWrSvlnTWl5xQtBdw63Hcx9LWkAuCPAAARwElEQVS6/NnXGxZSbyM9wO8ozcj7bkZy2G28dGcGVdVreHLZZlwOG7dndmn8B4Vohvq51LD4QOC3BUKdT8EMnm02xfO3DqGyupbn/roNl8PGhIt6BB1TpJLCFQBvb2gN/HPbIZ8/Y/cx8rP7uLnq703ta5Rm5QeV4xw2Zo0ZxtSFxfz8nY3EO+3ckN7Z7LBElAvmysbK+QR1sxkv3pFOpaeGGf/7GfFOO3dndTU7LFMEVbiUUrcDM4D+QJbWOir3Dm/OG9pb0fL3ui/+Cpqv+242pejx6Iem3/NyOe3k5WYyfl4RD769nji7jdGDOpoSS6SIlZwKpeZe2YR6hwYjOO02Xh2TwbSFa3j8z5twOW3cnJFmdlhhF2xX4WbgFuATA2KxrOZ0BKX6+Fr914N9oNlbtyTUFcf6Dzn+YtkmU1baB0iIszN3wggGp7bmx2+u5V+fHwzbuSNUROWUWbs4hIJRCx+E+ncS77AzO3c4OT3a8vDSDfxl035Djx8JgipcWuutWmvr9WYbrDlv6MZ+xogn/huuTOBtGrLCU8PiVXtMXVkgKd7BgklZXNihJdMXreH/dhwO27kjTSTlVKSsWhEoI1b6CNfvxOW0kz8+k2FdL+D+N9fxj60HDD2+1SkjngtQSv0L+Km/aQ2l1DRgGkDXrl2H7969O+jzhpOvNll/7bP+vnbRcyu8TkukJid803XYVD0e/fC8VeV9CeY8zXW0rIq78lZScqyChZOyyOzeJqznN5JSao3WOjOEx/8XFs+pULyH/bHSox++hPt3ctLt4Z78QrbtP0XBhEwu6dPO8HOEU6B51eg9LqXU3wFvNyae0Fq/F2hAWus8IA8gMzMz4p6i8zZv3lj7rL+5dl/3x85uHNmcpPQ1R9+U84dSmxZxvDElmztnr2LivNUsnprNkLTksMdhtmjJqVA1MwTSYWuFRz+8CXeDRyuXk4WTsrgrbxVTFxazYGIW2T3bhuRcVtLoVKHW+iqt9SAvHwEnWLTy1T77k7fXNzq37e/+WHOnF3ztotzU84dS+5YuFk/JpnWik9yCIrbuP2lKHGaKlpwKxbYlvqbaZry/JSJWkTFjK5fkxLoBYWpyApPmr2bdnmMhO5dVxMSST6HibxTVWPHx1VhxVnOS0tsc/dicrqZtjOlL5+QE3pyaQ4LTzj35hew4eMq0WETzhWLTVV+DweMV3re0t0qr+llmbUSbkhTPkqk5pLSMZ/zcIjaXngjp+cwWVOFSSt2slCoBRgIfKqWWGxNWZGhsFOWv+NQvMr40JynP7qK867lr+fTRK3jmpsGmb4zpTZc2iSyZmo1SijFzCvnqcJmp8VhFJOWUUc0M9TvwAp3qPstKrepg7ka0HVrVzWa0dDnJLSjkiwPROyA0pDmjqTIzM3VxceQ/nuLtKf2GFLDruWv9HifcN3St5POvT3FX3koS4xws/cFIv4XcSkLdnNFUkZhT3vJHgdcGowsSnbg9teetiGGFQZjVfHW4jDtmr6RWw9LpOfRsl2R2SAELNK9kqjAIgVw1BTIi9DW9cHm/dgE/DxKpz9P07diSRZOzOen2MGbOKg6cdJsdkggTXyvSNLwvm+C089T1Ay05c2BF3VNasGRqNlprxuYXsvdoudkhGU6uuAzia420QJOrYSfV5f3a8c6a0oCOF+y5rWDtnmPk5hfSsbWLt6ePJCUp3uyQ/JIrruD5e3wjNTnB0m3vkeCzfSe5e84qWiU4WDp9JJ1aW382I9C8ksJlIKOeM1m2rpSHl27wujyUt+nDaJlqXLXzCBPmFdG9bQvempZDcmKc2SH5JIUreNHyvrWyDXuPMza/kPYt43lreg7tW7rMDskvmSo0QcPGiOYWrcfe3eRzTUNvDRtWXxw0UDk925KXm8nOQ2WMm1vESbf3TjIRHczqwIsl6V2SmTdxBPtPuMnNL+JoWZXZIRlCCpfFeJv3r8/bPbNQPjsS7ntnl17Yjj+OHcZn+04yad5qyiqrQ3o+YR4zO/BiyYjubSgYn8muI2XkFhRywsejBZFECpfF+LtK8jUaDdXI1ay16K4a0IGX78pg7Z5jTF1YjLuZO0UL6zNilkI07ju9U5idO5wvDpxiwrwiTkf4gFAKl8X4ukqyK+VzNBqqkau/jfVC7dohnfj9Hems3HmEH7yxhspqKV5CBOPyvu159e5hbCw5weT5q6moitycko0kDWREc8Yjo/o2q0MwmN1VfTH73tnNGWm4PbU89u4m7n9zHbPGDMNpl7GWEM01elBHXrwjnZ+8vZ5pi4qZMy4Tl58VfKxK/goYxKhpNaOvnoK5R2XGumsN3Z3VlaeuH8DyLQfqOi1rI259ZiEs5cahqTx/6xD+s/0w9y1Zi6em1uyQmkyuuAzib1qtqUXHqKunxlavb4yvq79wd31NvKgHbk8tz3+0jXiHjedvHYLN5mv5YCFEY+7I7EKlp4Yn39vCT95az8t3DcURQbMZUrgMYva0mjfBFtOz32OFPZB++N1euD01vPyP7bicdp6+cSDKy8aZQojA5I7sTmV1Lc98uJV4h40Xbk+PmAGhFC4/mnLPytdeWGYuAmpEMQ3FvbPm+slVfXB7apj9yU5cThuPf7+/FC8hgjDlkp5UVNXw+4+/IN5p59mbB0VETknh8qGp02xWmVarL1zFNFw70yqlePSafrg9Ncz5zy4SnHYe+p48rCpEMO67ojcVnhr++K8viXfYeOr6AZYvXlK4fGjqNFsop9WaWxjCUUyDvY/WVEopnrp+IG5PLa+s2EG80869l/c2/DxCxAqlFI+M6ovbU8vcT3eREGfnZ6P6Wrp4SeHyoTnTbKGYVgumMITjHpWRTSmBstkUz94yGHd13XlcTjuTL+4RknMJEQuUUjx5XX/c1TW89q8vSXDauf/KPmaH5ZMULh+scs/KiAaLUN6jMqspxW5T/P72dKqqa/n1B5/hctoYm90tpOcUIpoppXjmxkG4PTW8+PEXuJw2pl3ay+ywvIqc/scws8oCoFbsVqzPzGe9HHYbL9+VwRX92vOLZZt5Z01JyM8pjBepe8lFI5tN8btbh3DtkE48+5dtLFz5ldkheRVU4VJKzVRKbVNKbVRK/VkplWxUYGazygKgVngI2B+zC3ycw8Yfxw7jol4pPPI/G/hg476wnDdUojmnvDFrPUzhm8Nu46U7h3L1gA788r0tLF291+yQzhPsFdfHwCCt9RDgC+Cx4EOyDissAGp2YWiMFQq8y2knb9xwhne7gJ+8tZ6/bfk6bOcOgajOqYbMXA9T+Oa025g1JoNL+qTw83c38t56aw0kgrrHpbX+W71PVwG3BReOaMhKDwH7YoVnvRLjHMydMIJ7Coq4b8k65ozP5LIL25kaU3PEWk5ZfSo8lsU77OTlZjJxfhEPLd1AvMPG6EGdzA4LMPYe1yTgr76+qJSappQqVkoVHzp0yMDTho9Zc/FWuPKLBC1dThZOzKJ3+ySmLSxm5ZdHzA4pWFGfU1afCo91CXF28sePID2tNT9+cx3/3HbQ7JCAAAqXUurvSqnNXj5urPc9TwDVwGJfx9Fa52mtM7XWme3aRd5IWObiI0PrRCeLJmfRtU0ikxesZs3uY2aHdB7JqW9ZfSpcQFK8g3kTs+jbsSXT31jDpzsOmx1S44VLa32V1nqQl4/3AJRS44HrgLFa+9hvPgrIXHzkaJsUz+Ip2bRvGc+EuUVsKjlhdkjnkJz6lhXukYrGtU5wsmhSNj3atmDKgmJWf3XU1HiC7SocDfwcuEFrXW5MSNYkc/GRpX0rF4un5tAqwUnu3EK2fX3S7JACEks5dZZMhUeGC1rE8caUbDolu5g4bzXr9x43LZZg73HNAloCHyul1iulXjcgJkuSufjIk5qcwJtTc4h32Lgnv5AvD502O6RAxExOicjTrmU8S6bk0KZFHOMKCtmyz5zZjKAKl9a6t9a6i9Z66JmPHxgVmNXIXHxk6to2kcVTcgAYO6eQPUesfRETSzklIlPH1i4WT8kmKd5BbkER2w+cCnsMsnJGgGQuPnL1bp/EG1OycVfXcPecVTK9K0SQurRJZPHUHOw2xZj8QnYdLgvr+ZUZ934zMzN1cXFx2M8rYtumkhOMmbOKtklxLJ0+kvatXM0+llJqjdY608DwgiI5Jcyw/cAp7sxbhcth4+3pI+nSJjGo4wWaV3LFJWLG4LTWzJ80goOnKhmbX8iR05VmhyREROvToSWLJmdxurKasfmFfH3CHZbzSuESMWV4tzYUjB/BnqPl5BYUcaLcY3ZIQkS0gZ1bs3ByNkfLqhiTv4pDp0I/IJTCJWLOyF5tyRuXyY6Dpxk3r4hTbileQgRjaJdk5k4Ywf7jbnILCjlWVhXS80nhEjHpsgvb8Yexw9hSeoLJ84spr6o2OyQhIlpWjzbMGZfJzsNljJtbxMkQDgilcImYdfWADrx011CKdx9l2sI1uBusjCKEaJqL+6Tw+j3D2Pb1SSbOW01ZZWgGhFK4REy7bkhnZt6Wzn93HOZHi9dSVV1rdkhCRLQr+nXglbsyWL/3OJMXrA7JgFAKl4h5tw5P4zc3D2LFtoM88NY6qmukeAkRjGsGd+L3t6dTuOso0xatobLa2OIlhUsIYGx2N568bgB/3fw1D/9pAzW1Ub22rRAhd1NGKs/dMphPvjjEfUvW4TFwQCiFS4gzJl/cg0dG9eW99ft44s+bqJXiJURQ7hzRlV/dMJCPPzvAg2+vN2xAKIVLiHruvbw3P76iN28X72XtHuvt5SVEpBn/ne48dk0/Pti4n/9sN2bDU4chRxEiijx09YVcPaADQ9KSzQ5FiKgw/bJejOzV1rCckisuIRpQSknREsJgRuaUFC4hhBARRQqXEEKIiBJU4VJK/VoptfHMTq1/U0p1NiowIWKR5JQQjQv2imum1nqI1noo8AHwSwNiEiKWSU4J0YigCpfW+mS9T1sA8uCLEEGQnBKicUG3wyulfgOMA04Al/v5vmnAtDOfnlZKfR7suQOUAhwO07kaY6VYQOLxp7FYuoXqxJJTTWKlWEDi8SeQWALKK6W1/wGdUurvQEcvX3pCa/1eve97DHBprZ8K5MThopQqtsoW61aKBSQef0IZi+SUcawUC0g8/hgZS6NXXFrrqwI81hLgQ8BSSSaE1UhOCRGcYLsK+9T79AZgW3DhCBHbJKeEaFyw97ieU0r1BWqB3cAPgg/JcHlmB1CPlWIBiccfs2KRnGoaK8UCEo8/hsXS6D0uIYQQwkpk5QwhhBARRQqXEEKIiBL1hUspNVMpte3MMjp/VkqZuuy3Uup2pdQWpVStUsq0NlWl1Gil1OdKqR1KqUfNiuNMLHOVUgeVUpvNjONMLF2UUv9USm098//pAbNjshrJKZ9xSE55j8XwnIr6wgV8DAzSWg8BvgAeMzmezcAtwCdmBaCUsgN/AK4BBgB3K6UGmBUPMB8YbeL566sGHtZa9wdygHtN/t1YkeRUA5JTfhmeU1FfuLTWf9NaV5/5dBWQZnI8W7XW4VrhwJcsYIfWeqfWugp4C7jRrGC01p8AR806f31a6/1a67Vn/vsUsBVINTcqa5Gc8kpyyodQ5FTUF64GJgF/NTsIC0gF9tb7vAT543wepVR3IAMoNDcSS5OcqiM5FQCjcirotQqtIJAldJRST1B3ybrYCvGYTHl5TZ6LqEcplQS8A/ykwcK3MUFyqskkpxphZE5FReFqbAkdpdR44DrgSh2GB9easKSPWUqALvU+TwP2mRSL5SilnNQl2GKt9btmx2MGyakmk5zyw+icivqpQqXUaODnwA1a63Kz47GI1UAfpVQPpVQccBfwvskxWYJSSgEFwFat9Ytmx2NFklNeSU75EIqcivrCBcwCWgIfn9lV9nUzg1FK3ayUKgFGAh8qpZaHO4YzN9bvA5ZTd6N0qdZ6S7jjOEsp9SawEuirlCpRSk02KxbgIiAXuOLM+2W9Uur7JsZjRZJTDUhO+WV4TsmST0IIISJKLFxxCSGEiCJSuIQQQkQUKVxCCCEiihQuIYQQEUUKlxBCiIgihUsIIUREkcIlhBAiovx/w0pKTU3h2b8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "##########################\n",
    "### 2D Decision Boundary\n",
    "##########################\n",
    "\n",
    "w, b = model2.linear.weight.view(-1), model2.linear.bias\n",
    "\n",
    "x_min = -2\n",
    "y_min = ( (-(w[0] * x_min) - b[0]) \n",
    "          / w[1] )\n",
    "\n",
    "x_max = 2\n",
    "y_max = ( (-(w[0] * x_max) - b[0]) \n",
    "          / w[1] )\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3))\n",
    "ax[0].plot([x_min, x_max], [y_min, y_max])\n",
    "ax[1].plot([x_min, x_max], [y_min, y_max])\n",
    "\n",
    "ax[0].scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')\n",
    "ax[0].scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')\n",
    "\n",
    "ax[1].scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], label='class 0', marker='o')\n",
    "ax[1].scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], label='class 1', marker='s')\n",
    "\n",
    "ax[1].legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matplotlib 3.0.2\n",
      "numpy      1.15.4\n",
      "torch      1.0.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}