softmax_confidence.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from src.Datasets import load\n",
    "from torch.utils.data import DataLoader\n",
    "from src.model import Conv_Net_Dropout, Conv_Net_FC_Dropout, Conv_Net\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Conv_Net_FC_Dropout(\n",
       "  (convs): Sequential(\n",
       "    (0): Conv2d(1, 16, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
       "    (1): ReLU()\n",
       "    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (3): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): ReLU()\n",
       "    (5): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (6): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
       "    (7): ReLU()\n",
       "  )\n",
       "  (fcs): Sequential(\n",
       "    (0): Dropout(p=0.5, inplace=False)\n",
       "    (1): Linear(in_features=3136, out_features=128, bias=True)\n",
       "    (2): ReLU()\n",
       "    (3): Dropout(p=0.5, inplace=False)\n",
       "    (4): Linear(in_features=128, out_features=10, bias=True)\n",
       "    (5): Softmax(dim=1)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Conv_Net_FC_Dropout()\n",
    "model.load_state_dict(torch.load('/home/julius/OOD/models/model3_drop_fc_0_5.pth'))\n",
    "model.to('cuda')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "mnist_dataset = load('MNIST')(train = False)\n",
    "mnist_loader = DataLoader(mnist_dataset, batch_size = 128, shuffle = False)\n",
    "\n",
    "cifar_dataset = load('CIFAR10')(train = False)\n",
    "cifar_loader = DataLoader(cifar_dataset, batch_size = 128, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 79/79 [00:00<00:00, 96.17it/s] \n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    mnist_softmax = []\n",
    "    for x, y in tqdm(mnist_loader):\n",
    "        x = x.to('cuda')\n",
    "        out = model(x)\n",
    "        mnist_softmax.append(torch.max(out.cpu(), dim = 1)[0])\n",
    "    mnist_softmax = torch.cat(mnist_softmax).numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 79/79 [00:01<00:00, 50.07it/s]\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    cifar_softmax = []\n",
    "    for x, y in tqdm(cifar_loader):\n",
    "        x = x.to('cuda')\n",
    "        out = model(x)\n",
    "        cifar_softmax.append(torch.max(out.cpu(), dim = 1)[0])\n",
    "    cifar_softmax = torch.cat(cifar_softmax).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MNIST min and max: 0.43137366 1.0\n",
      "CIFAR min and max: 0.15096828 1.0\n"
     ]
    }
   ],
   "source": [
    "print('MNIST min and max:', np.min(mnist_softmax), np.max(mnist_softmax))\n",
    "print('CIFAR min and max:', np.min(cifar_softmax), np.max(cifar_softmax))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGdCAYAAAD0e7I1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8HklEQVR4nO3deXxU1f3/8ffMZGVJYgSSUIMEQVFBcAMjWoumjXWlUpdvEUFRtID+kLphFSouLFWK2AhflUUeX5SvWrBWLRZjga8KiCBWKyIIAgoJWiSBAFlm7u+PyUwyk5nkzjDJ3Ju8no/HPJLce2fm5IrMm3M+5xyHYRiGAAAAWilnvBsAAADQnAg7AACgVSPsAACAVo2wAwAAWjXCDgAAaNUIOwAAoFUj7AAAgFaNsAMAAFq1hHg3wAo8Ho/27Nmjjh07yuFwxLs5AADABMMwdPDgQXXt2lVOZ/j+G8KOpD179ig3NzfezQAAAFHYvXu3TjjhhLDnCTuSOnbsKMl7s9LS0uLcGgAAYEZ5eblyc3P9n+PhEHYk/9BVWloaYQcAAJtpqgSFAmUAANCqEXYAAECrRtgBAACtGjU7JrndblVXV8e7Ga2ey+VSQkICSwAAAGKGsGPCoUOH9O2338owjHg3pU1o166dcnJylJSUFO+mAABaAcJOE9xut7799lu1a9dOnTt3psehGRmGoaqqKn3//ffasWOHevXq1egiUQAAmEHYaUJ1dbUMw1Dnzp2Vmpoa7+a0eqmpqUpMTNTOnTtVVVWllJSUeDcJAGBz/LPZJHp0Wg69OQCAWOJTBQAAtGqEHQAA0KoRdgAAQKtG2GmlRo4cKYfDoTvuuKPBubFjx8rhcGjkyJEB106bNi3gutdffz2gVmnlypVyOBw6cOCA/9jzzz+vfv36qUOHDsrIyNCZZ56pqVOnSpK6d+8uh8MR9uF7fwAAmhNhpxXLzc3VkiVLdOTIEf+xo0eP6qWXXlK3bt0Crk1JSdH06dP1448/mn79+fPna/z48brrrru0adMmffDBB7rvvvt06NAhSdL69eu1d+9e7d27V3/5y18kSVu2bPEfe/rpp2PwWwIA0DimnkfIMAwdqXbH5b1TE10RzQo766yz9PXXX2vp0qUaNmyYJGnp0qXq1q2b8vLyAq4tKCjQtm3bNHXqVM2YMcPU67/xxhu67rrrNGrUKP+x008/3f99586d/d9nZmZKkrp06aKMjAzTvwMAoM6WkoOaXbxVd/+8l3p26Rjv5tgGYSdCR6rdOm3SO3F57y+mFKpdUmT/yW655RYtWLDAH3bmz5+vm2++WStXrgy4zuVy6YknntBvfvMb3XXXXTrhhBOafO3s7GytWrVKO3fu1IknnhhRuwAAkbv+uTU6cLha63bs18cPFcS7ObbBMFYrd+ONN+r999/Xzp07tXPnTn3wwQe68cYbQ177q1/9Sv3799fkyZNNvfbkyZOVkZGh7t2765RTTtHIkSP1yiuvyOPxxPJXAADUOnDYu0fjD4cq49wSe6FnJ0KpiS59MaUwbu8dqc6dO+vyyy/XwoULZRiGLr/8cnXq1Cns9dOnT9fFF1+se+65p8nXzsnJ0Zo1a/T5559r9erV+vDDDzVixAi98MILWr58OYsDAgAsgbATIYfDEfFQUrzdcsstGjdunCSpqKio0Wt/+tOfqrCwUBMnTjQ9W6pPnz7q06ePxowZozvuuEMXXnihVq1apcGDBx9r0wEAOGb2+tRGVC699FJVVVXJ4XCosLDpXqlp06apf//+OuWUUyJ+r9NOO02SVFFREfFzAQBoDnEdZ1i9erWuvPJKde3aVQ6HQ6+//nrAecMwNGnSJOXk5Cg1NVUFBQXaunVrwDX79+/XsGHDlJaWpoyMDI0aNco/9RleLpdLmzdv1hdffCGXq+mhsL59+2rYsGGaPXt2o9f99re/1aOPPqoPPvhAO3fu1Nq1a3XTTTepc+fOys/Pj1XzAQA4JnENOxUVFerXr1/YoZUZM2Zo9uzZmjt3rtatW6f27dursLBQR48e9V8zbNgw/fvf/9aKFSv05ptvavXq1Ro9enRL/Qq2kZaWprS0NNPXT5kypclC44KCAq1du1bXXnutTj75ZA0dOlQpKSkqLi7W8ccff6xNBgAgJhyGYRjxboTkrYVZtmyZhgwZIsnbq9O1a1f97ne/8xfLlpWVKSsrSwsXLtQNN9ygzZs367TTTtP69et1zjnnSJKWL1+uyy67TN9++626du1q6r3Ly8uVnp6usrKyBoHg6NGj2rFjh/Ly8pSSkhK7Xxhhcc8BILTuD7zl//6baZfHsSXW0Njnd32WnS6zY8cOlZSUqKCgbh2B9PR0DRw4UGvWrJEkrVmzRhkZGf6gI3l7G5xOp9atWxf2tSsrK1VeXh7wAAAArZNlw05JSYkkKSsrK+B4VlaW/1xJSYm6dOkScD4hIUGZmZn+a0KZOnWq0tPT/Y/c3NwYtx4AAFiFZcNOc5o4caLKysr8j927d8e7SQAAoJlYNuxkZ2dLkkpLSwOOl5aW+s9lZ2dr3759Aedramq0f/9+/zWhJCcn+wt2Iy3cBQAA9mLZsJOXl6fs7GwVFxf7j5WXl2vdunX+ac35+fk6cOCANmzY4L/mvffek8fj0cCBA1u8zQAAwHriuqjgoUOHtG3bNv/PO3bs0KZNm5SZmalu3bpp/Pjxeuyxx9SrVy/l5eXp4YcfVteuXf0ztk499VRdeumluu222zR37lxVV1dr3LhxuuGGG0zPxAIAAK1bXMPOxx9/HLClwIQJEyRJI0aM0MKFC3XfffepoqJCo0eP1oEDB3TBBRdo+fLlAdORFy9erHHjxumSSy6R0+nU0KFDm1wMDwAAO3I4JGssGGMvcQ07P/vZz9TYMj8Oh0NTpkzRlClTwl6TmZmpl156qTmaBwCApTgdDrlJOxGzbM0OAAAI5HTEuwX2RNhp5UpKSnTnnXeqR48eSk5OVm5urq688kp/4Xf37t01a9Ys//Xdu3eXw+EIeJxwwgkBr1lYWCiXy6X169c3eL+RI0f6n5eYmKi8vDzdd999AVt8AACi43SQdqLBruet2DfffKNBgwYpIyNDf/zjH9W3b19VV1frnXfe0dixY/Xll1+GfN6UKVN02223+X+uv3norl279OGHH2rcuHGaP3++zj333AbPv/TSS7VgwQJVV1drw4YNGjFihBwOh6ZPnx77XxIA2hDCTnQIO63YmDFj5HA49NFHH6l9+/b+46effrpuueWWsM/r2LFj2HWKFixYoCuuuEK//e1vdd5552nmzJlKTU0NuCY5Odn//NzcXBUUFGjFihWEHQA4RgxjRYewEynDkKoPx+e9E9t5S/FN2L9/v5YvX67HH388IOj4ZGRkRPz2hmFowYIFKioqUu/evdWzZ0+99tprGj58eNjnfP755/rwww914oknRvx+AIBA9OxEh7ATqerD0hNxWsPnwT1SUsPgEsq2bdtkGIZ69+4d8dvcf//9euihh/w/P/HEE7rrrrv07rvv6vDhwyosLJQk3XjjjZo3b16DsPPmm2+qQ4cOqqmpUWVlpZxOp/785z9H3A4AQCCyTnQIO61UY1P6m3Lvvfdq5MiR/p87deokSZo/f76uv/56JSR4/9j813/9l+699159/fXXOumkk/zXDx48WHPmzFFFRYX+9Kc/KSEhQUOHDo26PQAALyfjWFEh7EQqsZ23hyVe721Sr1695HA4whYhN6ZTp07q2bNnwLH9+/dr2bJlqq6u1pw5c/zH3W635s+fr8cff9x/rH379v7nz58/X/369dO8efM0atSoiNsCAKjDMFZ0mHoeKYfDO5QUj0cEf8gzMzNVWFiooqIiVVRUNDh/4MCBiH7txYsX64QTTtCnn36qTZs2+R9PPfWUFi5cKLfbHfJ5TqdTDz74oB566CEdOXIkovcEAASq37FzLD34bQ1hpxUrKiqS2+3WgAED9Je//EVbt27V5s2bNXv2bP9mqmbNmzdPv/71r9WnT5+Ax6hRo/TDDz9o+fLlYZ977bXXyuVyqaio6Fh/JQBo0xz1/tHr9hB2zCLstGI9evTQxo0bNXjwYP3ud79Tnz599POf/1zFxcUBQ1FN2bBhgz799NOQdTfp6em65JJLNG/evLDPT0hI0Lhx4zRjxoyQvUwAAHPq9+zUEHZMcxj0g6m8vFzp6ekqKytTWlpawLmjR49qx44dysvLC9iAFM2Hew4AoZ0/tVh7yrwr0n/+SKE6JLft0tvGPr/ro2cHAACbqD+MVeP2xLEl9kLYAQDAhqrdbX5gxjTCDgAANuGpV3lS46FnxyzCDgAANhEQdujZMY2wAwCATdQv06mmZsc0wo5JTFprOdxrAAgtcBiLvyvNIuw0weVySZKqqqri3JK24/Bh767yiYmJcW4JAFhL/YUEGcYyr21P0DchISFB7dq10/fff6/ExEQ5neTD5mIYhg4fPqx9+/YpIyPDHzQBAF4eDwXK0SDsNMHhcCgnJ0c7duzQzp07492cNiEjI0PZ2dnxbgYAWE79YSymnptH2DEhKSlJvXr1YiirBSQmJtKjAwBhuANmY9GzYxZhxySn08nWBQCAuKo/ckWBsnkUoAAAYBPugGEsenbMIuwAAGATzMaKDmEHAAAbCF6DjNlY5hF2AACwAbcnOOzQs2MWYQcAABtwB/fsMIxlGmEHAAAbCB61okDZPMIOAAA20KBnh2Es0wg7AADYgKfBMBY9O2YRdgAAsAFPUE8O20WYR9gBAMAGGs7GomfHLMIOAAA2QM1O9Ag7AADYQHBHDlPPzSPsAABgAxQoR4+wAwCADQTX7FQzjGUaYQcAABugZyd6hB0AAGygQc8ONTumEXYAALCBBj07TD03jbADAIANBJfoMBvLPMIOAAA20HBRQcKOWYQdAABsoEHYoUDZNMIOAAA2EFyzw9Rz8wg7AADYAD070SPsAABgAxQoR4+wAwCADTCMFT3CDgAANsAwVvQIOwAA2ICHqedRI+wAAGADbvbGihphBwAAG2hQoEzPjmmEHQAAbCB4GCu4hgfhEXYAALCB4HBD1jGPsAMAgA0E1+wYBmnHLMIOAAA2EDyMFbzuDsIj7AAAYAPBw1YMY5lH2AEAwAZ8w1gJTockenYiQdgBAMAGfMNYCS5v2CHrmGfpsON2u/Xwww8rLy9PqampOumkk/Too48GFGUZhqFJkyYpJydHqampKigo0NatW+PYagAAYs83GyvR6f3opmfHPEuHnenTp2vOnDn685//rM2bN2v69OmaMWOGnnnmGf81M2bM0OzZszV37lytW7dO7du3V2FhoY4ePRrHlgMAEFu+YSyXi2GsSCXEuwGN+fDDD3X11Vfr8ssvlyR1795dL7/8sj766CNJ3l6dWbNm6aGHHtLVV18tSVq0aJGysrL0+uuv64Ybbohb2wEAiCXDX7NT27PDbhGmWbpn5/zzz1dxcbG++uorSdKnn36q999/X7/85S8lSTt27FBJSYkKCgr8z0lPT9fAgQO1Zs2asK9bWVmp8vLygAcAAFbm2wor0V+zQ8+OWZbu2XnggQdUXl6u3r17y+Vyye126/HHH9ewYcMkSSUlJZKkrKysgOdlZWX5z4UydepUPfLII83XcAAAYsw/G8s/jBXP1tiLpXt2XnnlFS1evFgvvfSSNm7cqBdffFFPPvmkXnzxxWN63YkTJ6qsrMz/2L17d4xaDABA8/BQoBw1S/fs3HvvvXrggQf8tTd9+/bVzp07NXXqVI0YMULZ2dmSpNLSUuXk5PifV1paqv79+4d93eTkZCUnJzdr2wEAiCXfbCyXk56dSFm6Z+fw4cNyOgOb6HK55KmtysrLy1N2draKi4v958vLy7Vu3Trl5+e3aFsBAGhOHv8wlvdzkZod8yzds3PllVfq8ccfV7du3XT66afrk08+0cyZM3XLLbdIkhwOh8aPH6/HHntMvXr1Ul5enh5++GF17dpVQ4YMiW/jAQCIIV/YSWTqecQsHXaeeeYZPfzwwxozZoz27dunrl276vbbb9ekSZP819x3332qqKjQ6NGjdeDAAV1wwQVavny5UlJS4thyAABiyzcbK4FhrIg5DPrBVF5ervT0dJWVlSktLS3ezQEAoIGif27TH9/ZogF5mfpox351TEnQZ38ojHez4srs57ela3YAAICXf7sI9saKGGEHAAAb8NXouJh6HjHCDgAANlC3zo63Z8dN0Y5phB0AAGwgeAVlOnbMI+wAAGAD/tlYLoaxIkXYAQDABvzr7DhZZydShB0AAGzA4wkuUI5na+yFsAMAgA24g1ZQltgywizCDgAANuCfjeWq++imd8ccwg4AADYQPBtLom7HLMIOAAA24JuNFdizQ9gxg7ADAIANGP4VlOvX7MSrNfZC2AEAwAbcQSsoS/TsmEXYAQDABupqdihQjhRhBwAAG/DNxqJAOXKEHQAAbMBdm2sSnXUf3YYnTo2xGcIOAAA24AlRoEzPjjmEHQAAbKBuUUHCTqQIOwAA2IBvNpbT6ZCjNu9QoGwOYQcAABvwD2M5HHLWph32xjKHsAMAgA3U79lx0rMTEcIOAAA24As2LodDjtqeHWp2zCHsAABgA75g43SqXs8OYccMwg4AADbgH8YKqNmJZ4vsg7ADAIAN+MKOy1kXdujZMYewAwCADdSfjcXU88gQdgAAsAFfsHHSsxMxwg4AADYQWLPjPcY6O+YQdgAAsIG6vbFUr2cnni2yD8IOAAA2UL9nh3V2IkPYAQDABgJnYwUeQ+MIOwAA2IBRbwVl1tmJDGEHAAAb8A1ZORwOVlCOEGEHAAAb8G8X4VC9mp14tsg+CDsAANiAUX+dndpPb3p2zCHsAABgA/V7dupqdgg7ZhB2AACwAXdAzQ7DWJEg7AAAYAMej/drwN5YpB1TCDsAANiAYdTfLoKenUgQdgAAsAFfsHE4xN5YESLsAABgAx56dqJG2AEAwAb8Yccp9saKEGEHAAAb8PXiOFlBOWKEHQAAbKD+MJbLyd5YkSDsAABgA75p5oHbRZB2zCDsAABgA0bIYaz4tcdOCDsAANhA6NlYpB0zCDsAANhA3XYRrLMTKcIOAAA24Km367mDdXYiQtgBAMAGfL04LqaeR4ywAwCADdStsyNWUI4QYQcAABvw+Gt26gqUqdkxh7ADAIDFGYZRb+q5t0hZYhjLLMIOAAAWV3+4KmDquSdODbIZwg4AABZXvweHvbEiR9gBAMDiAsKOU/VqduLVInsh7AAAYHFG0DAWe2NFxvJh57vvvtONN96o448/Xqmpqerbt68+/vhj/3nDMDRp0iTl5OQoNTVVBQUF2rp1axxbDABAbIUbxnITdkyxdNj58ccfNWjQICUmJurvf/+7vvjiCz311FM67rjj/NfMmDFDs2fP1ty5c7Vu3Tq1b99ehYWFOnr0aBxbDgBA7LjrVSg7WGcnYgnRPGn79u3q0aNHrNvSwPTp05Wbm6sFCxb4j+Xl5fm/NwxDs2bN0kMPPaSrr75akrRo0SJlZWXp9ddf1w033NDsbQQAoLk1mI1V21XBOjvmRNWz07NnTw0ePFj/8z//06w9KG+88YbOOeccXXvtterSpYvOPPNMPf/88/7zO3bsUElJiQoKCvzH0tPTNXDgQK1Zs6bZ2gUAQEsyAoaxVFezQ9eOKVGFnY0bN+qMM87QhAkTlJ2drdtvv10fffRRrNum7du3a86cOerVq5feeecd/fa3v9Vdd92lF198UZJUUlIiScrKygp4XlZWlv9cKJWVlSovLw94AABgVfUzjcvpYBgrQlGFnf79++vpp5/Wnj17NH/+fO3du1cXXHCB+vTpo5kzZ+r777+PSeM8Ho/OOussPfHEEzrzzDM1evRo3XbbbZo7d+4xve7UqVOVnp7uf+Tm5sakvQAANIf6BcoO1tmJ2DEVKCckJOiaa67Rq6++qunTp2vbtm265557lJubq5tuukl79+49psbl5OTotNNOCzh26qmnateuXZKk7OxsSVJpaWnANaWlpf5zoUycOFFlZWX+x+7du4+pnQAANCdfqPGFHNbZicwxhZ2PP/5YY8aMUU5OjmbOnKl77rlHX3/9tVasWKE9e/b4i4ajNWjQIG3ZsiXg2FdffaUTTzxRkrdYOTs7W8XFxf7z5eXlWrdunfLz88O+bnJystLS0gIeAABYlW9bCF/IYW+syEQ1G2vmzJlasGCBtmzZossuu0yLFi3SZZddJmdteXheXp4WLlyo7t27H1Pj7r77bp1//vl64okndN111+mjjz7Sc889p+eee06Stytv/Pjxeuyxx9SrVy/l5eXp4YcfVteuXTVkyJBjem8AAKyirmfHEfCVmh1zogo7c+bM0S233KKRI0cqJycn5DVdunTRvHnzjqlx5557rpYtW6aJEydqypQpysvL06xZszRs2DD/Nffdd58qKio0evRoHThwQBdccIGWL1+ulJSUY3pvAACswhd2HP5hrMDjaFxUYWfFihXq1q2bvyfHxzAM7d69W926dVNSUpJGjBhxzA284oordMUVV4Q973A4NGXKFE2ZMuWY3wsAACvyZRqXM7Bnh3V2zImqZuekk07SDz/80OD4/v37Axb9AwAAxy54GMvBMFZEogo74ZLkoUOHGD4CACDGfKGGYazoRDSMNWHCBEneRDlp0iS1a9fOf87tdmvdunXq379/TBsIAEBb59sbiwLl6EQUdj755BNJ3p6dzz77TElJSf5zSUlJ6tevn+65557YthAAgDbOaLDOTuBxNC6isPPPf/5TknTzzTfr6aefZn0aAABagK8Hp2HNDmHHjKhmY9XfhRwAADQvf4Gyk2GsaJgOO9dcc40WLlyotLQ0XXPNNY1eu3Tp0mNuGAAA8Gq4XUTgcTTOdNhJT0/3d5ulp6c3W4MAAEAgI2gYy9fDQ9Yxx3TYqT90xTAWAAAtJ+xsLMaxTIlqnZ0jR47o8OHD/p937typWbNm6R//+EfMGgYAALzCbxcRpwbZTFRh5+qrr9aiRYskSQcOHNCAAQP01FNP6eqrr9acOXNi2kAAANq64NlYTmZjRSSqsLNx40ZdeOGFkqTXXntN2dnZ2rlzpxYtWqTZs2fHtIEAALR1vvV06vbGCjyOxkUVdg4fPqyOHTtKkv7xj3/ommuukdPp1HnnnaedO3fGtIEAALR1wdtFsDdWZKIKOz179tTrr7+u3bt365133tEvfvELSdK+fftYaBAAgBgLv10EaceMqMLOpEmTdM8996h79+4aOHCg8vPzJXl7ec4888yYNhAAgLYu3HYR9OyYE9UKyr/+9a91wQUXaO/everXr5//+CWXXKJf/epXMWscAAAIUaDsX2eHtGNGVGFHkrKzs5WdnR1wbMCAAcfcIAAAEKhu6rlvb6zA42hcVGGnoqJC06ZNU3Fxsfbt2yePxxNwfvv27TFpHAAAqAs1rtriE18Pj9sT7hmoL6qwc+utt2rVqlUaPny4cnJy/EkTAADEXoPtIph6HpGows7f//53vfXWWxo0aFCs2wMAAIL4ZmM5mI0VlahmYx133HHKzMyMdVsAAEAIwbues85OZKIKO48++qgmTZoUsD8WAABoHg23i/AdJ+2YEdUw1lNPPaWvv/5aWVlZ6t69uxITEwPOb9y4MSaNAwAAodbZ8U09j1eL7CWqsDNkyJAYNwMAAIRDz86xiSrsTJ48OdbtAAAAYdTV7PjW2aFAORJR1exI0oEDB/TCCy9o4sSJ2r9/vyTv8NV3330Xs8YBAIB6YSdonR0KlM2JqmfnX//6lwoKCpSenq5vvvlGt912mzIzM7V06VLt2rVLixYtinU7AQBos4J7dlhnJzJR9exMmDBBI0eO1NatW5WSkuI/ftlll2n16tUxaxwAAJB8GxU0XGcnXi2yl6jCzvr163X77bc3OP6Tn/xEJSUlx9woAABQp+E6O4HH0biowk5ycrLKy8sbHP/qq6/UuXPnY24UAACo48s0Lnp2ohJV2Lnqqqs0ZcoUVVdXS/J2q+3atUv333+/hg4dGtMGAgDQ1gXveu4rVKZmx5yows5TTz2lQ4cOqXPnzjpy5Iguuugi9ezZUx07dtTjjz8e6zYCANCmucMsKsgwljlRzcZKT0/XihUr9MEHH+jTTz/VoUOHdNZZZ6mgoCDW7QMAoM0LXlTQv86OJ14tspeIw47H49HChQu1dOlSffPNN3I4HMrLy1N2drYMw/D/BwAAALFhNFhnx/uVnh1zIhrGMgxDV111lW699VZ999136tu3r04//XTt3LlTI0eO1K9+9avmaicAAG2WxxNUs8PeWBGJqGdn4cKFWr16tYqLizV48OCAc++9956GDBmiRYsW6aabboppIwEAaMs8DWZj+Y6TdsyIqGfn5Zdf1oMPPtgg6EjSxRdfrAceeECLFy+OWeMAAECodXYoUI5ERGHnX//6ly699NKw53/5y1/q008/PeZGAQCAOg23i2CdnUhEFHb279+vrKyssOezsrL0448/HnOjAABAHV+ocbA3VlQiCjtut1sJCeHLfFwul2pqao65UQAAoE7wMBY9O5GJqEDZMAyNHDlSycnJIc9XVlbGpFEAAKCO0WCdHe/P1OyYE1HYGTFiRJPXMBMLAIDY8k09dzqp2YlGRGFnwYIFzdUOAAAQRvB2ES6nb50d0o4ZUe2NBQAAWk7D7SJ8xwk7ZhB2AACwOIMC5WNC2AEAwOJ8PTjB20V4SDumEHYAALC44GEstouIDGEHAACL84UaV+2ntoNhrIgQdgAAsDj/1HN6dqJC2AEAwOIabhfhm3oerxbZC2EHAACLC79dBGnHDMIOAAAWx3YRx4awAwCAxbER6LEh7AAAYHH+sOPbG6v205vtIswh7AAAYHFuj/erM3hRQbKOKYQdAAAsruF2Ed6v1OyYQ9gBAMDigreLcLBdREQIOwAAWFzD7SJYZycStgo706ZNk8Ph0Pjx4/3Hjh49qrFjx+r4449Xhw4dNHToUJWWlsavkQAAxFjD2ViBx9E424Sd9evX67//+791xhlnBBy/++679be//U2vvvqqVq1apT179uiaa66JUysBAIg933CVy0mBcjRsEXYOHTqkYcOG6fnnn9dxxx3nP15WVqZ58+Zp5syZuvjii3X22WdrwYIF+vDDD7V27do4thgAgNgJ3i6CRQUjY4uwM3bsWF1++eUqKCgIOL5hwwZVV1cHHO/du7e6deumNWvWhH29yspKlZeXBzwAALCqcIsKknXMSYh3A5qyZMkSbdy4UevXr29wrqSkRElJScrIyAg4npWVpZKSkrCvOXXqVD3yyCOxbioAAM0ieLsI9saKjKV7dnbv3q3/9//+nxYvXqyUlJSYve7EiRNVVlbmf+zevTtmrw0AQKxRoHxsLB12NmzYoH379umss85SQkKCEhIStGrVKs2ePVsJCQnKyspSVVWVDhw4EPC80tJSZWdnh33d5ORkpaWlBTwAALCq4O0iHBQoR8TSw1iXXHKJPvvss4BjN998s3r37q37779fubm5SkxMVHFxsYYOHSpJ2rJli3bt2qX8/Px4NBkAgJhruF1E3TnDMPzhB6FZOux07NhRffr0CTjWvn17HX/88f7jo0aN0oQJE5SZmam0tDTdeeedys/P13nnnRePJgMAEHMNt4uoCzceQ3KRdRpl6bBjxp/+9Cc5nU4NHTpUlZWVKiws1LPPPhvvZgEAEDPB20UEhh1DLpF2GmO7sLNy5cqAn1NSUlRUVKSioqL4NAgAgGYWvF2Ew1n/HIU7TbF0gTIAAAi/zo7EWjtmEHYAALA4X6Cp2y6i7hw9O00j7AAAYHFuT2M1O3Fpkq0QdgAAsLjgYSwHPTsRIewAAGBx4baLkOp2REd4hB0AACwuuGfHVS/suAk7TSLsAABgcQ3W2XE6/ENZhJ2mEXYAALA4X56p36Pj+95NzU6TCDsAAFhc3Uagdcd809Dp2WkaYQcAAIsLHsaSCDuRIOwAAGBxnqBdzyXCTiQIOwAAWFzwbCyJsBMJwg4AABYXvM6ORIFyJAg7AABYnNvfs8MwVjQIOwAAWBzDWMeGsAMAgMX5h7HqpR1fLw9hp2mEHQAALC5Uz06CyxFwDuERdgAAsLiQ6+zUfl/jJuw0hbADAIDFNbrODj07TSLsAABgcb6eHRezsaJC2AEAwOLqhrHqjlGgbB5hBwAAi/OEWFSQAmXzCDsAAFicEWLXcycFyqYRdgAAsLiQPTtOenbMIuwAAGBxodbZ8S0wWEPNTpMIOwAAWJyvCDnkRqCEnSYRdgAAsLhQu55ToGweYQcAAIvzhNj1nAJl8wg7AABYXKh1dihQNo+wAwCAxXlC7XpOgbJphB0AACzO42k4G8tXoOwh7DSJsAMAgMWF3BvLxWwsswg7AABYnC/POEJMPWcYq2mEHQAALMyoV4DspEA5KoQdAAAsrH7HTcDUc6dvGKulW2Q/hB0AACzME9CzE2oFZdJOUwg7AABYWP0CZEe9T+26AuWWbpH9EHYAALCw+iU5Lnp2okLYAQDAwsIOY/lqdihQbhJhBwAAC6sfdupvF+GiQNk0wg4AABYWbjZWXdgh7TSFsAMAgIWFW2eHnh3zCDsAAFhY/dlYTD2PDmEHAAALCxjGclKgHA3CDgAAFlZT23OTUH8MSwxjRYKwAwCAhdW4vT03Ca5wYYe00xTCDgAAFubb1TzRGfiRTc+OeYQdAAAsrKY2zbiCe3Yc7HpuFmEHAAALq/YNY4Xp2anxEHaaQtgBAMDCfFPPE8PU7HgIO00i7AAAYGHVtQXIrqDZWE5/zw5FO00h7AAAYGG+2ViJrsCP7AQKlE0j7AAAYGG+AuUG6+xQoGwaYQcAAAvzFSAnuChQjhZhBwAAC2tqBWUKlJtG2AEAwMKqw6ygTIGyeYQdAAAszB1mBeUEf89OizfJdgg7AABYWLU7zNRzB7uem2XpsDN16lSde+656tixo7p06aIhQ4Zoy5YtAdccPXpUY8eO1fHHH68OHTpo6NChKi0tjVOLAQCIrXAbgSZQoGyapcPOqlWrNHbsWK1du1YrVqxQdXW1fvGLX6iiosJ/zd13362//e1vevXVV7Vq1Srt2bNH11xzTRxbDQBA7PhqcoLX2aFA2byEeDegMcuXLw/4eeHCherSpYs2bNign/70pyorK9O8efP00ksv6eKLL5YkLViwQKeeeqrWrl2r8847Lx7NBgAgZvxTz8OuoEzYaYqle3aClZWVSZIyMzMlSRs2bFB1dbUKCgr81/Tu3VvdunXTmjVrwr5OZWWlysvLAx4AAFhRU8NY9Ow0zTZhx+PxaPz48Ro0aJD69OkjSSopKVFSUpIyMjICrs3KylJJSUnY15o6darS09P9j9zc3OZsOgAAUav2r6Ac+JFNgbJ5tgk7Y8eO1eeff64lS5Yc82tNnDhRZWVl/sfu3btj0EIAAGLP7QnTs+NyBJxHeJau2fEZN26c3nzzTa1evVonnHCC/3h2draqqqp04MCBgN6d0tJSZWdnh3295ORkJScnN2eTAQCIibA1Ow7CjlmW7tkxDEPjxo3TsmXL9N577ykvLy/g/Nlnn63ExEQVFxf7j23ZskW7du1Sfn5+SzcXAICY8w9jhZmNRdhpmqV7dsaOHauXXnpJf/3rX9WxY0d/HU56erpSU1OVnp6uUaNGacKECcrMzFRaWpruvPNO5efnMxMLANAq1K2gHLpAmbDTNEuHnTlz5kiSfvaznwUcX7BggUaOHClJ+tOf/iSn06mhQ4eqsrJShYWFevbZZ1u4pQAANI+6vbEoUI6WpcOOYeI/YEpKioqKilRUVNQCLQIAoGXVuEPvek6BsnmWrtkBAKCtqwkzG4sCZfMIOwAAWJhvu4jgdXbYLsI8wg4AABbmX0E5TIEy20U0jbADAICFhS1QdlKgbBZhBwAAC3P7dz0P7NlxUbNjGmEHAAALqw6zgjKLCppH2AEAwMJ8U89dYVZQlihSbgphBwAACwu3gnL9sEORcuMIOwAAWFi4AuWAnh2KlBtF2AEAwMLq1tkJXaDsvYaw0xjCDgAAFlbXsxN+GIsi5cYRdgAAsDC3fzYWBcrRIuwAAGBhvtlYwevs1B/VYhircYQdAAAszDeM5Qqq2XE4HHX7Y1Gg3CjCDgAAFuafeu5q+JHNKsrmEHYAALCw6jCzsSRWUTaLsAMAgIXVhJmNJRF2zCLsAABgYb4C5eDZWFJd2KFAuXGEHQAALMwXZBrr2aFAuXGEHQAALKymkQJlJwXKphB2AACwsGrfruchCpQTqNkxhbADAICF1e16Hr5mh7DTOMIOAAAWZmY2FgXKjSPsAABgYWbW2aFAuXGEHQAALMrjMeTLMQkhC5S9XxnGahxhBwAAi/L16kihh7F8a+8QdhpH2AEAwKJ89TpS6AJlJwXKphB2AACwqPphp9Gp59TsNIqwAwCARdXUG8ZKDDGM5e/ZcRN2GkPYAQDAonxTyl1OhxyOELOxfAXK9Ow0irADAIBFNbZ6skSBslmEHQAALKpu9eTQYScxwXu8qsYT8jy8CDsAAFhUtX/15NAf1+2SEiRJFVU1LdYmOyLsAABgUb4C5VDFyZLUIdkbdg5XulusTXZE2AEAwKJ8U8/D1ey0S3JJomenKYQdAAAsyjcbKyHEgoKS1L62Z6eikrDTGMIOAAAWVeNufBirvb9mh2GsxhB2AACwqOomhrHaJ3uHsQ7Ts9Mowg4AABbln3oeZjaWbxjrEAXKjSLsAABgUb5dz0PteC7VFSgfpkC5UYQdAAAsyjcbK2yBchIFymYQdgAAsCi3r2cnbM0OBcpmEHYAALCouhWUKVA+FoQdAAAs6mi1t8cmKcEV8rxvu4hDhJ1GEXYAALCoHw9XSZIy2yWGPO/fLqLKLcNg5/NwCDsAAFjUfypqw0775JDn29UOY9V4DFWy83lYCfFuAAAAbZZhSD9+I+35RCr5TDqwUyr7VjpaJrmrNepQlX6RlKLjvukqvXu2lN1Hyj1PSv+JJKldYt3w1uEqt1ISQw93tXWEHQAAWpJhSN9tlDa/IW3+m7T/67CXdpHUxSnpP1ul91fVO3GadMovlXDGDUpJdOpotUcVlTXKbJ/U7M23I8IOAADNzV0j7VrjDTdfvimVf1d3zpno7bHJ6Scd30tKP0FKPU5yJenBpZ/qP9/v1fiBHXWqc7e091NpzyZp3xfex/89pVdcJ2uu+1JVHD1fUrt4/YaWRtgBAKA51FRK21d5e3C2vC0d/k/ducT20sm/kE69Uur1Cym5Y8iXeL/6qHZ5cnVbv3ype6b34OH90rZi6fPXpK0rdIbxlZ5N+kpHl/xNuuhuqd8NUkLoGp+2irADAECsVB6Str3r7cH56h2p6mDdudTjpFMu8wacHoOlxJQmX26/v0C53vBUu0zpjGu9j0P79D/PTNIVR99QRvkO6W93SSunSuffJZ1zs5SYGuvf0JYIOwAAHIsDu6VtK6StK6Sv35Nqjtad65AtnXqFN+CcOEhyhZ5CHkpljdu/fs7xYWZjqUMXLcu4SU/s/Ln+MuArnbpjkXRwj/TOROnD2dJP75HOvElKaNu1PIQdAAAicXi/9O166Zv/k7a+K32/OfB8xonSaVdJp14l/eQcKcy+Vk3x9eokOB1KSw3/cd0+OUGHlaIvThyuU6/6nfTpy9LqP0plu6W3fie9/7T0s/ulM26QXG3zY79t/tYAAJhx5Edp35feYuDvNkq713lnRtXncEonnCv1/Ll0cqGU3VdyhN7eIRL/OeQNO8e1T5KjkddrX7vzeUVVjbcH5+wR3rqdjYuk1U9KZbukv46V/m+m9LOJUp9rJGfbmqJO2AEAtF01lVLF91LZd96ekAO7vI8fd3hDzqGS0M87vpeUO1DqebG3/qZdZsyb5uvZOb6J6eTt/Duf19sMNCFZGnCbdOaN0vp50vszvVPcl94q/d9T0uAHvUNrMQhldkDYAQDYm2F462SOlnsX46us/er/vtz79fB/vMGm4ofar997r2lK2glSl97eqeG5A729OM0QboKFLE4OoYNvM9CqEPtjJaZK54/z9vas+29vHc/3m6VXhnvX6jn3VumM66XkDjFvv5UQdgBYl8cjGW7JUyO5q71fPbU/G+6gi4P+hRrwL9ZmPtfYv44bnGvseTE+ZxiS4Ql6hDoW6TW+8+7Q5z1h/pt5aiRPddDPNd41aDwhHtVHah+HvWHG973/eO2j5oj3+mg5XFLaT6SMXCk91/s140Spc2+p8ylSSlr0r30M/mMy7LRLNrEZaHJHb7HyubdKa4qktc96h+bemiCtmCz1/y/vuc6nxKz9VtJqwk5RUZH++Mc/qqSkRP369dMzzzyjAQMGxLtZQPMxDO9f8DWVkrvK+8HirvR+9R+r/6gO+jBp7AMp1M9mrvH9bPY1m/jZYK8fRMLhDSbJ6bVf06SUet+nHid16CK17yS17yy1r/0+JSPqIuLmtL+iUpKZnp3azUArg/8BEEJqhnTx76X8sdKml6T1L3iHtz56zvvI6S+dPsQ7Rb7Tya1mmKtVhJ3//d//1YQJEzR37lwNHDhQs2bNUmFhobZs2aIuXbrEu3mwM3dNbVAIFyJCHKupvTbkscqg59U7VhMcTuofq/ea9a9TG93l2JngLQr192gE3YeA3Z9jdK61cTgbeTiiPy+Hd8aP0/dI9BbD+n52Bf3c2PmEVO8wTMCjnZSQ4v3qO5aQ4u25SOpgydASrW37DklqOuxcdHJnpacm6uSs0AsThpSaIeWPkQbeIe1Y6a3r2fJ3ae8m7+PdP3iH7/Iu9Aagrv29hddJ7aP6XeLNYbSCPeEHDhyoc889V3/+858lSR6PR7m5ubrzzjv1wAMPNPn88vJypaenq6ysTGlp8emubLV83d31hx487jDHar963EE9FVVB4SHM9wFhoTrMtcHBoomwYbeeBVeytzDRlSi5kuoeCcn1Pkh8HzL1P3BCPYLPh/nZ/5qhXq+x1wj+0IvgNeL9r836f202+CvUKueMxsMKWoxhGKrxGHJ7ar+6DVV7PP6fE50OpSa5lOhyqrLaoyPVbq3d/h+N/99NcjikZWMGqX9uRvM3tOIH72KIm9+QvvnA+3djfQ6ntzA7M8+7pUX6Cd5hvw5ZtT1otY/ktBYLnWY/v23fs1NVVaUNGzZo4sSJ/mNOp1MFBQVas2ZNyOdUVlaqsrLuP2JZmbdArby8PLaN++s4b4W/7y8j/19EJn5u8lo1cT7cv1jDXB/81cz7BwSXmnr1FfWCi2wWFpriTPQGCleid4qnK1Fy+sJFYmDQcCZJCfVDR2Pf1w8otV8TkgLfr/7rO30hpt7rOBP8H2Izln+pf31b5r39HknVx/Zrx+bfRIakGhk6htqKCDSoamniA77+79ggTtTPEw2fGPhj+FMyZO49GmtbE28f8B7B5xu+Z/j2yOTzgs8bhuSu/WD3GIacDoecDsnpdMjpcMjhkPervMcSnA65nA65fB+OhiGP4f09jNrX872fxzC8o7dG4DmPUXfO8J8zJHnf2+Hwvkf9drhq2+J7XW97pcpqtyprPKr2eJTgcCjB5ZTL6VCCy6FEp+97Z1CA8ajGbfh/7xqPRzUeQzUeyVN7LFrDBnZTj3Rn7D+fQkqSTh7qfVQd9k6z/26jdzf2kn9JFfuk7770PpriSq77x1dCsvfvqOsWSpk9Ytpi331p6u8o24edH374QW63W1lZWQHHs7Ky9OWXof+DTJ06VY888kiD47m5uc3SRgAAojGt9mE/Bxseuv/M5nu3gweVnp4e9rztw040Jk6cqAkTJvh/9ng82r9/vxITE9WtWzft3r2b4awYKi8vV25uLvc1hrinscc9jT3uafPgvtYxDEMHDx5U165dG73O9mGnU6dOcrlcKi0tDTheWlqq7OzskM9JTk5WcnLgPiMZGRn+7rC0tLQ2/weoOXBfY497Gnvc09jjnjYP7qtXYz06PrYvW09KStLZZ5+t4uJi/zGPx6Pi4mLl5+fHsWUAAMAKbN+zI0kTJkzQiBEjdM4552jAgAGaNWuWKioqdPPNN8e7aQAAIM5aRdi5/vrr9f3332vSpEkqKSlR//79tXz58gZFy01JTk7W5MmTGwxx4dhwX2OPexp73NPY4542D+5r5FrFOjsAAADh2L5mBwAAoDGEHQAA0KoRdgAAQKtG2AEAAK1amws7RUVF6t69u1JSUjRw4EB99NFHjV7/6quvqnfv3kpJSVHfvn319ttvt1BL7SOSe/r888/rwgsv1HHHHafjjjtOBQUFTf43aKsi/bPqs2TJEjkcDg0ZMqR5G2hDkd7TAwcOaOzYscrJyVFycrJOPvlk/g4IEuk9nTVrlk455RSlpqYqNzdXd999t44ePdpCrbW+1atX68orr1TXrl3lcDj0+uuvN/mclStX6qyzzlJycrJ69uyphQsXNns7bcdoQ5YsWWIkJSUZ8+fPN/79738bt912m5GRkWGUlpaGvP6DDz4wXC6XMWPGDOOLL74wHnroISMxMdH47LPPWrjl1hXpPf3Nb35jFBUVGZ988omxefNmY+TIkUZ6errx7bfftnDLrS3S++qzY8cO4yc/+Ylx4YUXGldffXXLNNYmIr2nlZWVxjnnnGNcdtllxvvvv2/s2LHDWLlypbFp06YWbrl1RXpPFy9ebCQnJxuLFy82duzYYbzzzjtGTk6Ocffdd7dwy63r7bffNn7/+98bS5cuNSQZy5Yta/T67du3G+3atTMmTJhgfPHFF8YzzzxjuFwuY/ny5S3TYJtoU2FnwIABxtixY/0/u91uo2vXrsbUqVNDXn/dddcZl19+ecCxgQMHGrfffnuzttNOIr2nwWpqaoyOHTsaL774YnM10Zaiua81NTXG+eefb7zwwgvGiBEjCDtBIr2nc+bMMXr06GFUVVW1VBNtJ9J7OnbsWOPiiy8OODZhwgRj0KBBzdpOuzITdu677z7j9NNPDzh2/fXXG4WFhc3YMvtpM8NYVVVV2rBhgwoKCvzHnE6nCgoKtGbNmpDPWbNmTcD1klRYWBj2+rYmmnsa7PDhw6qurlZmZmZzNdN2or2vU6ZMUZcuXTRq1KiWaKatRHNP33jjDeXn52vs2LHKyspSnz599MQTT8jtdrdUsy0tmnt6/vnna8OGDf6hru3bt+vtt9/WZZdd1iJtbo34nDKnVaygbMYPP/wgt9vdYFXlrKwsffnllyGfU1JSEvL6kpKSZmunnURzT4Pdf//96tq1a4P/WduyaO7r+++/r3nz5mnTpk0t0EL7ieaebt++Xe+9956GDRumt99+W9u2bdOYMWNUXV2tyZMnt0SzLS2ae/qb3/xGP/zwgy644AIZhqGamhrdcccdevDBB1uiya1SuM+p8vJyHTlyRKmpqXFqmbW0mZ4dWM+0adO0ZMkSLVu2TCkpKfFujm0dPHhQw4cP1/PPP69OnTrFuzmthsfjUZcuXfTcc8/p7LPP1vXXX6/f//73mjt3brybZlsrV67UE088oWeffVYbN27U0qVL9dZbb+nRRx+Nd9PQyrWZnp1OnTrJ5XKptLQ04Hhpaamys7NDPic7Ozui69uaaO6pz5NPPqlp06bp3Xff1RlnnNGczbSdSO/r119/rW+++UZXXnml/5jH45EkJSQkaMuWLTrppJOat9EWF82f1ZycHCUmJsrlcvmPnXrqqSopKVFVVZWSkpKatc1WF809ffjhhzV8+HDdeuutkqS+ffuqoqJCo0eP1u9//3s5nfz7O1LhPqfS0tLo1amnzfzJSkpK0tlnn63i4mL/MY/Ho+LiYuXn54d8Tn5+fsD1krRixYqw17c10dxTSZoxY4YeffRRLV++XOecc05LNNVWIr2vvXv31meffaZNmzb5H1dddZUGDx6sTZs2KTc3tyWbb0nR/FkdNGiQtm3b5g+OkvTVV18pJyenzQcdKbp7evjw4QaBxhcmDbZpjAqfUybFu0K6JS1ZssRITk42Fi5caHzxxRfG6NGjjYyMDKOkpMQwDMMYPny48cADD/iv/+CDD4yEhATjySefNDZv3mxMnjyZqedBIr2n06ZNM5KSkozXXnvN2Lt3r/9x8ODBeP0KlhTpfQ3GbKyGIr2nu3btMjp27GiMGzfO2LJli/Hmm28aXbp0MR577LF4/QqWE+k9nTx5stGxY0fj5ZdfNrZv32784x//ME466STjuuuui9evYDkHDx40PvnkE+OTTz4xJBkzZ840PvnkE2Pnzp2GYRjGAw88YAwfPtx/vW/q+b333mts3rzZKCoqYup5CG0q7BiGYTzzzDNGt27djKSkJGPAgAHG2rVr/ecuuugiY8SIEQHXv/LKK8bJJ59sJCUlGaeffrrx1ltvtXCLrS+Se3riiScakho8Jk+e3PINt7hI/6zWR9gJLdJ7+uGHHxoDBw40kpOTjR49ehiPP/64UVNT08KttrZI7ml1dbXxhz/8wTjppJOMlJQUIzc31xgzZozx448/tnzDLeqf//xnyL8jffdxxIgRxkUXXdTgOf379zeSkpKMHj16GAsWLGjxdludwzDoOwQAAK1Xm6nZAQAAbRNhBwAAtGqEHQAA0KoRdgAAQKtG2AEAAK0aYQcAALRqhB0AANCqEXYAAECrRtgBAACtGmEHAAC0aoQdAADQqhF2AABAq/b/AXT7aVqAIRW9AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.kdeplot(mnist_softmax, label = 'MNIST')\n",
    "sns.kdeplot(cifar_softmax, label = 'CIFAR')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7dde36314e00>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsG0lEQVR4nO3dfXBV5aHv8d9+yd4hQAIYkwDGRrAKikIFyY2W8dhJm1aH1nunU0Z7gUMVj5Wea8mcVhAltbSE41UOvZWWK5XaztRC66i3p2Tw2LScjiUdrrycYyviRVAomEBEskMCSfZez/0j+xUCZofs9Zis72cmk2RlrbWf/ZT6/PaznhefMcYIAADAEr/tAgAAAG8jjAAAAKsIIwAAwCrCCAAAsIowAgAArCKMAAAAqwgjAADAKsIIAACwKmi7AP3hOI6OHTum0aNHy+fz2S4OAADoB2OM2tvbNWHCBPn9F+7/GBJh5NixYyovL7ddDAAAMABHjhzRFVdcccG/D4kwMnr0aEm9b6awsNByaQAAQH9EIhGVl5cn2/ELGRJhJPFoprCwkDACAMAQ81FDLBjACgAArCKMAAAAqwgjAADAKsIIAACwijACAACsIowAAACrCCMAAMAqwggAALCKMAIAAKzKOoz88Y9/1Ny5czVhwgT5fD69/PLLH3nN9u3bddNNNykcDuvqq6/Wc889N4CiAgCA4SjrMNLR0aHp06dr/fr1/Tr/0KFDuvPOO3X77bdr7969+uY3v6n77rtPr7zyStaFBQAAw0/We9N84Qtf0Be+8IV+n79hwwZdddVVeuqppyRJU6dO1WuvvaZ/+Zd/UU1NTbYvDwAAhpmcb5TX1NSk6urqjGM1NTX65je/ecFrurq61NXVlfw9EonkqngAAEnGGMUcI8dIXdGYHCd+XEbGJH7uPc8kr+n9e+KP5x4/97rE39Sf8zLONWo/G5WRFI0ZRWOOumKOTkS6VBAOyBjJSbu/kZHjpJU37XWMes9N/Jz8uzE61Nqh0qL888qZXkfnvYe095KoL513LOMm5x0799qL3T/jNfo878Kv39c90o/d++mrVD6uQDbkPIw0NzertLQ041hpaakikYjOnDmjESNGnHdNfX29Hn/88VwXDcAQYkxvQ+kkG02jM90xxRyjWPyYMUr+zYmfH3OMImd65Pf7FI31Ho86Ro7T+/1Q62mNzs9Lnm9M798Sr5Vo6FK/G73dclrjx+TLcYxijhRznN57GqP/ONKmiuICOY4Ui98rlrje6S3nrsMf6pMlo+TzxRvPtIax971mNp6J92+kZKOf+D3jXJN5rjFSc+SsCvODfbwPZdQT8KUZE4ZvGBmI5cuXq7a2Nvl7JBJReXm5xRIBuZP4RBqNN1TRmFFnT7S3UU00aGkNZKKxjTlGLZGzygv6pXMamGQjFf+5r0+C6ceOt59V0O+TY6S/HG1TaWH+OY1c6p7p91fyE2jinvFPXGmfQN/4W5uuKh6pqGPUE3N0qrNH75w4rU9cVpD8JB5zTMb7Ot7e2zMaDvrTwoed/30G4s33P7o396/H3OvxjZyN5uzePp/kS/7sS/tZ8in1R1/8WO/PvrSfM6/TufeL/9J+tvf/E1cVj1TQ71Mw4FdPzFHr6S5NLSvsfT1f6t6Je/rTfvad87M/fv/0ch460aHrJxQm30OivOnvN/176l0o4z2dd35f98g4L+OGGff3nXP/9GszjsV/6aNo/bpHaWG+bMl5GCkrK1NLS0vGsZaWFhUWFvbZKyJJ4XBY4XA410XDMBCNOeqKOqlPoIlPoWmfSju6o71du/HGsDva+x+w/LxAxqdtc94nxsyG2zHSsVNnNCIUUHPbWUXO9ujgiQ6NKciTMVLUcZKh4sjJTnX1OLpsVChVnmTZehvdDzt7bFefaw62dpx37O2W0x95XVfU6df98wI++X29XwF/bwMT8KeOdfXE1N4V1dUloxSMHw8Ges81Rvrbh52a+YmxyfMTDZU/2WD54o1a6vf3PujQDROL5Pf7FPT33ivg8ykQ8Ol4pEuTS0b1/u5XRtn88fM6uqK6fHQ42fhmNJiJBlWK/z31e6KBTjXu5/ztnHso/ntBKJj2fpRRV4nyJRruvIBPeQH/hUPGOQ0ncKlyHkaqqqrU0NCQcezVV19VVVVVrl8aLjPGqCdm1Hq6S2d7Yvqgo1vHTp3pDQuOUXfUUUvkrDq7Y/p/x9t12ciwuqIx7XrvlCaOHSFjes+JOkYHjp9W0Yg8Bfy+eA+BSeshULJb/uOuvevSPpEWhAIK+HobMH+8gfX5ehszv0/q7IkpcqZH0yYWnddQJT/19fFJ0J/4BJVogCQdO3VWV5eM0tiReTpy8oyum1CYeZ+M+yfuk2qYMs/LfI3W012afPmoZCMnSXkBv0bnB5OhIdGY+/2pIFEQCqQd951XF3kBf/J+AIaurMPI6dOndeDAgeTvhw4d0t69ezVu3DhdeeWVWr58uY4ePaqf//znkqQHHnhATz/9tL797W/ra1/7mn7/+9/rV7/6lbZu3Tp47wID0n62R6e7ouqJGvU4jjq7YjrdFVVnd1RHT51Ra3uXoo7RyY5uHT11RqPCQb37Qad8knpivaGhO+roWNsZBf0+9cQGHg5aT3edd6ztzMB6Dnw+ZTReMceoO+Zo4pgRycawoyuqzp5Y73N7pX069Cvj9/RPyFLv98MnOzWjfIx8Pp8K84MaEQqofGyBQkG/8gI+Bfz+eH04KhqRp7ygPx4eUg1tevlG5QeTn6zzAn4F4p+0+fQJwCuyDiOvv/66br/99uTvibEdCxcu1HPPPaf3339fhw8fTv79qquu0tatW7V06VL94Ac/0BVXXKGf/OQnTOsdZDHH6FRnt1oiXWo706N3P+iQT9JfjrXpg9PdajvTo5bIWX3Q0a3Orpi6Y/3r/u6vc4NIKOhX0Yg8nWjvUsnosCZfPkrjRoaUF/DpbI+jwhFB5QX8mjq+UPl5AZ3pjmri2BHKC/gV9Pc26pI0Oj8v2c2d7HZP/wTt8/V2P8d/DtCIA8CQ4zPp84A+piKRiIqKitTW1qbCwkLbxXFV25kevfdBR2/AaO3Q4ZOd8vl8+u1/HNPo/Dztb2lXKOC/pHBREArEu7t9aj3drYrLClQQCirqOCotzNfEMSM0piCkoN+niuKRGjMiTwWhgIIBv4IBn/L8vd3to/KDyvP7NSre9Q4A8Lb+tt8fy9k0XuI4RkdPndHhk5061dmj3Yc/1N8+7FTTOx989Aj4trOS1GcQufGKIjW3ndXsq8ap7UyPJhWP1JWXjdTlo8O6bGRIpYX5GjcypHEjQ7l4WwAA9BthxEU9MUc7D53Uv/7HMb1z4rRaIl06fLLzI68bFQ7qdFdUN105RidOd2l80QjNrhinUflB3VwxTiPDAV0+KqzR+XkKBRnMBwAYWggjOdR2pkc7DrTqpzve1c5DJ5MzQy5mRvkYjSnI0xemlam0MF+fKh+rooI8l0oMAID7CCODyHGM/nosol+9fkT/ebRN+45FMh6hJIJIzfWlqrhspIIBn267pkQzysfQowEA8CzCyCBobjurH28/oJ81vXfe3yYU5WtWxTiNGxnSbdderjlXFyvIuggAACQRRi7BgePtevxf39SfD36QMbX1M1NKNPMTY/VfJo3Tp8rHys/MEgAALogwMgAHT5zW//73g/r1riMZ+2Ws/q836Mszr+CRCwAAWSCMZKH1dJf+57b9emH335LjP6qnluprt1aoavJlLLYFAMAAEEb6wRijX+48osf/9a/JjbtumXyZln72Gt1cMc5y6QAAGNoIIx/hTHdMD/5il/6w/4Sk3tVK199zk/7u2svpCQEAYBAQRi7iTHdMX3vu/6rp4AeSpLtnl6tu7vXKzwtYLhkAAMMHYeQCjDH6xvO71XTwA40KB/W/7p6hz0wptV0sAACGHcLIBby4+6ga3zqucNCvny66mbEhAADkCHNQ+/B2S7u++9s3JUnfrGaQKgAAuUQYOceZ7pgWbtqptjM9mlE+RvfNucp2kQAAGNYII+f497dP6P22syotDOu5RTcrj6XbAQDIKVrac2z7y/uSpLk3TtCYgpDl0gAAMPwRRtJ0RWNq3HdckvT5aWWWSwMAgDcQRtLseOcDtXdFVTI6rJuuHGu7OAAAeAJhJM22N5olSTXXl7HTLgAALiGMpHn9vZOSpM9MKbFcEgAAvIMwkqY71rsJXlFBnuWSAADgHYSRNLGYkSQF2AAPAADXEEbSxEw8jDBeBAAA1xBG0sSf0hBGAABwEWEkjUPPCAAAriOMpInGu0b8jBkBAMA1hJE0Tm/HiIL0jAAA4BrCSJqYw2MaAADcRhhJk5hNw+qrAAC4hzCSJtkzwpgRAABcQxhJw2MaAADcRxiJcxKjV0UYAQDATYSRuMR4EYnHNAAAuIkwEhdL6xnxUysAALiGZjcuPYwESSMAALiGVjcu/TENWQQAAPfQ7MZlDGBlzAgAAK4hjMRFmU0DAIAVhJG4RM+I3yf56BkBAMA1hJG4xJgRekUAAHAXYSQuluwZIYwAAOAmwkgcS8EDAGAHYSSOMAIAgB2EkTiHMSMAAFhBGImLOb3fWWMEAAB3EUbiok5vGvHTMwIAgKsII3HxLKIgYQQAAFcRRuIS64wwtRcAAHcRRuKYTQMAgB2EkTjCCAAAdhBG4ggjAADYQRiJS64zwpgRAABcRRiJS+5NQ88IAACuIozEpR7TWC4IAAAeQ9MblwojVAkAAG6i5Y2LJceMWC4IAAAeM6Awsn79elVUVCg/P1+VlZXauXPnRc9ft26drr32Wo0YMULl5eVaunSpzp49O6AC54rDbBoAAKzIOoxs2bJFtbW1qqur0+7duzV9+nTV1NTo+PHjfZ7//PPPa9myZaqrq9O+ffv07LPPasuWLXrkkUcuufCDKeqwAisAADZkHUbWrl2rxYsXa9GiRbruuuu0YcMGFRQUaNOmTX2ev2PHDt1666265557VFFRoc997nO6++67P7I3xW2Jqb1BntMAAOCqrMJId3e3du3aperq6tQN/H5VV1erqampz2tuueUW7dq1Kxk+Dh48qIaGBt1xxx0XfJ2uri5FIpGMr1yL0TMCAIAVwWxObm1tVSwWU2lpacbx0tJSvfXWW31ec88996i1tVWf/vSnZYxRNBrVAw88cNHHNPX19Xr88cezKdolYwVWAADsyPlsmu3bt2v16tX60Y9+pN27d+vFF1/U1q1btWrVqgtes3z5crW1tSW/jhw5kutipsIIPSMAALgqq56R4uJiBQIBtbS0ZBxvaWlRWVlZn9c89thjmj9/vu677z5J0g033KCOjg7df//9WrFihfx9rOsRDocVDoezKdolS07tpWcEAABXZdUzEgqFNHPmTDU2NiaPOY6jxsZGVVVV9XlNZ2fneYEjEAhIkkw8AHwcMLUXAAA7suoZkaTa2lotXLhQs2bN0uzZs7Vu3Tp1dHRo0aJFkqQFCxZo4sSJqq+vlyTNnTtXa9eu1ac+9SlVVlbqwIEDeuyxxzR37txkKPk4YG8aAADsyDqMzJs3TydOnNDKlSvV3NysGTNmaNu2bclBrYcPH87oCXn00Ufl8/n06KOP6ujRo7r88ss1d+5cff/73x+8dzEIoowZAQDACp/5OD0ruYBIJKKioiK1tbWpsLAwJ6/xzB/f0eqGt/TfPjVRa+fNyMlrAADgJf1tv9mbJi7m9H7nMQ0AAO4ijMQ5hsc0AADYQBiJi8YYwAoAgA2EkbjEOiNBwggAAK4ijMSxzggAAHYQRuISPSNslAcAgLsII3GpjfIsFwQAAI+h6Y1LhRGqBAAAN9HyxtEzAgCAHTS9cawzAgCAHYSRuCgb5QEAYAVhJC4xtZd1RgAAcBdhJC5GzwgAAFYQRuJijBkBAMAKwkhcjBVYAQCwgjASRxgBAMAOwkhccmovYQQAAFcRRuKSA1gZMwIAgKsII3E8pgEAwA7CSBxhBAAAOwgjcbHeLMLUXgAAXEYYiXPoGQEAwArCSFzUcSSxAisAAG4jjMTFswh70wAA4DLCSFxiOXim9gIA4C7CSByzaQAAsIMwEpcKI5YLAgCAx9D0xrECKwAAdhBG4hJ70wT9VAkAAG6i5Y1L9oxQIwAAuIqmNy45ZoTHNAAAuIowEpeY2stsGgAA3EUYiWNqLwAAdhBG4tibBgAAOwgjcVGm9gIAYAVhJM5hzAgAAFYQRuISY0bYKA8AAHcRRuJS64wQRgAAcBNhJI51RgAAsIMwEsc6IwAA2EEYiXOc3u+EEQAA3EUYiaNnBAAAOwgjkowxqQGsjBkBAMBVhBFJ8RwiiZ4RAADcRhhRaiaNRBgBAMBthBGlVl+VCCMAALiNMKLUvjQS64wAAOA2wogyH9P4qREAAFxF0yvJSQsjQdIIAACuouVVao0RSWLICAAA7iKMKG2TPJ/kY8wIAACuIowobZM8ukUAAHAdYUSEEQAAbCKMKLXOCNN6AQBwH2FEqXVG/PSMAADgOsKIUlN7eUwDAID7BhRG1q9fr4qKCuXn56uyslI7d+686PmnTp3SkiVLNH78eIXDYV1zzTVqaGgYUIFzITG1N0gYAQDAdcFsL9iyZYtqa2u1YcMGVVZWat26daqpqdH+/ftVUlJy3vnd3d367Gc/q5KSEr3wwguaOHGi3nvvPY0ZM2Ywyj8oUlN7CSMAALgt6zCydu1aLV68WIsWLZIkbdiwQVu3btWmTZu0bNmy887ftGmTTp48qR07digvL0+SVFFRcWmlHmTMpgEAwJ6sHtN0d3dr165dqq6uTt3A71d1dbWampr6vOY3v/mNqqqqtGTJEpWWlmratGlavXq1YrHYBV+nq6tLkUgk4yuX6BkBAMCerMJIa2urYrGYSktLM46Xlpaqubm5z2sOHjyoF154QbFYTA0NDXrsscf01FNP6Xvf+94FX6e+vl5FRUXJr/Ly8myKmbXE1N5ggDACAIDbcj6bxnEclZSU6JlnntHMmTM1b948rVixQhs2bLjgNcuXL1dbW1vy68iRIzktY8zp/c46IwAAuC+rMSPFxcUKBAJqaWnJON7S0qKysrI+rxk/frzy8vIUCASSx6ZOnarm5mZ1d3crFAqdd004HFY4HM6maJck6vSmEdYZAQDAfVn1jIRCIc2cOVONjY3JY47jqLGxUVVVVX1ec+utt+rAgQNy4g2+JL399tsaP358n0HEBoeeEQAArMn6MU1tba02btyon/3sZ9q3b5++/vWvq6OjIzm7ZsGCBVq+fHny/K9//es6efKkHnroIb399tvaunWrVq9erSVLlgzeu7hEiXVGmE0DAID7sp7aO2/ePJ04cUIrV65Uc3OzZsyYoW3btiUHtR4+fFh+fyrjlJeX65VXXtHSpUt14403auLEiXrooYf08MMPD967uESswAoAgD0+Y+LdAh9jkUhERUVFamtrU2Fh4aDf/9U3W7T4569revkY/Z8ltw76/QEA8KL+tt/sTaO0Rc/oGAEAwHWEEaWtM+KnOgAAcButr9JWYKU2AABwHc2v2JsGAACbCCNibxoAAGwijCi1zkiQnhEAAFxHGBHrjAAAYBNhRFKUxzQAAFhDGFFqai89IwAAuI8wImbTAABgE2FEhBEAAGwijCh9OXjCCAAAbiOMKDW110/PCAAAriOMKDW1l3VGAABwH2FEUszp/U7PCAAA7iOMSIo5vWmEMSMAALiPMKLUmBFm0wAA4D7CiFKPaQgjAAC4jzAiVmAFAMAmwoikaIy9aQAAsIUwovSeEcsFAQDAg2h+lb4cPNUBAIDbaH2VNpuGxzQAALiOMCIpFuMxDQAAttD8ir1pAACwiTAi9qYBAMAmwojSekYYMwIAgOsII5KiDoueAQBgC2FEqcc0hBEAANxHGFH6OiOEEQAA3EYYUVoYYcwIAACuI4yIqb0AANhEGBE9IwAA2EQYUWqjvGCAMAIAgNsII5KiMdYZAQDAFsKIUj0jzKYBAMB9hBGlxozQMwIAgPsII5LiT2nYmwYAAAsII5JijiOJxzQAANhAGJEU680irDMCAIAFhBGl7U3DmBEAAFxHGFFqBVYe0wAA4D7CiNgoDwAAmwgjSg8jlgsCAIAH0fyKdUYAALCJMKK0vWn8VAcAAG6j9ZUUTfSMUBsAALiO5ldpU3sZwAoAgOsII0qb2suYEQAAXEcYEVN7AQCwiTAiwggAADYRRsTUXgAAbCKMKDW1l54RAADcRxhRqmckSBgBAMB1ng8jxhjFs4j8hBEAAFzn+TCS6BWRmNoLAIANAwoj69evV0VFhfLz81VZWamdO3f267rNmzfL5/PprrvuGsjL5kRijRGJnhEAAGzIOoxs2bJFtbW1qqur0+7duzV9+nTV1NTo+PHjF73u3Xff1T/90z9pzpw5Ay5sLjhO6mfGjAAA4L6sw8jatWu1ePFiLVq0SNddd502bNiggoICbdq06YLXxGIxffWrX9Xjjz+uSZMmXVKBB1s0LY0wmwYAAPdlFUa6u7u1a9cuVVdXp27g96u6ulpNTU0XvO673/2uSkpKdO+99/brdbq6uhSJRDK+ciW9Z4R1RgAAcF9WYaS1tVWxWEylpaUZx0tLS9Xc3NznNa+99pqeffZZbdy4sd+vU19fr6KiouRXeXl5NsXMSvqYEXpGAABwX05n07S3t2v+/PnauHGjiouL+33d8uXL1dbWlvw6cuRIzsqYPpuGLAIAgPuC2ZxcXFysQCCglpaWjOMtLS0qKys77/x33nlH7777rubOnZs85sSfiwSDQe3fv1+TJ08+77pwOKxwOJxN0QYsfV8aH49pAABwXVY9I6FQSDNnzlRjY2PymOM4amxsVFVV1XnnT5kyRW+88Yb27t2b/PriF7+o22+/XXv37s3p45f+SjymYY0RAADsyKpnRJJqa2u1cOFCzZo1S7Nnz9a6devU0dGhRYsWSZIWLFigiRMnqr6+Xvn5+Zo2bVrG9WPGjJGk847b4iQ2yfP88m8AANiRdRiZN2+eTpw4oZUrV6q5uVkzZszQtm3bkoNaDx8+LP8QatlT+9IMnTIDADCc+IxJm07yMRWJRFRUVKS2tjYVFhYO6r0PHD+t6rX/rsL8oP7zOzWDem8AALysv+2357sDHJMawAoAANzn+TCSPpsGAAC4jzBCGAEAwCrCiMPUXgAAbCKMmMTUXsIIAAA2eD6MODymAQDAKs+HEcaMAABgF2GEMSMAAFhFGGGdEQAArCKMJPamoWcEAAArPB9GEiuwBgOEEQAAbPB8GInG6BkBAMAmz4cR9qYBAMAuz4eRmNP7ndk0AADYQRihZwQAAKsII05v1whhBAAAOwgj8cc07E0DAIAdng8jyb1pyCIAAFjh+TCSGjPi+aoAAMAKz7fA0eRGeZYLAgCAR3m+CXbYtRcAAKs8H0bYmwYAALs8H0aSe9PQMwIAgBWeDyOJMSNM7QUAwA7Ph5FYcmovYQQAABs8H0YYwAoAgF2eDyPsTQMAgF2EEXpGAACwijDC1F4AAKwijPCYBgAAqzwfRhIDWFlnBAAAOzwfRlhnBAAAuzwfRhzWGQEAwCrPh5HEmBF6RgAAsIMw4vR+Z8wIAAB2EEac3jTCbBoAAOwgjMR7RlhnBAAAOzwfRpzkOiOWCwIAgEd5vglOLQfv+aoAAMAKz7fAyTDCUxoAAKwgjLBRHgAAVhFGWGcEAACrPB9G2JsGAAC7PB9GknvTMLUXAAArPB9GUlN7CSMAANjg+TDCAFYAAOwijBBGAACwijCSXGeEMAIAgA2EEab2AgBglefDiEPPCAAAVnk+jCR6RgKsBw8AgBWeDyPRGD0jAADY5PkwwjojAADY5fkwEmMFVgAArPJ8GIlnEQUZMwIAgBWeDyNRx5FEzwgAALYMKIysX79eFRUVys/PV2VlpXbu3HnBczdu3Kg5c+Zo7NixGjt2rKqrqy96vtviWYQxIwAAWJJ1GNmyZYtqa2tVV1en3bt3a/r06aqpqdHx48f7PH/79u26++679Yc//EFNTU0qLy/X5z73OR09evSSCz8YWIEVAAC7fMbEp5P0U2VlpW6++WY9/fTTkiTHcVReXq5//Md/1LJlyz7y+lgsprFjx+rpp5/WggUL+vWakUhERUVFamtrU2FhYTbF/Ug3f/93OtHepYb/MUfXTRjcewMA4GX9bb+z6hnp7u7Wrl27VF1dnbqB36/q6mo1NTX16x6dnZ3q6enRuHHjLnhOV1eXIpFIxleusFEeAAB2ZRVGWltbFYvFVFpamnG8tLRUzc3N/brHww8/rAkTJmQEmnPV19erqKgo+VVeXp5NMbOSCiM5ewkAAHARrjbBa9as0ebNm/XSSy8pPz//guctX75cbW1tya8jR47krEwO64wAAGBVMJuTi4uLFQgE1NLSknG8paVFZWVlF732ySef1Jo1a/S73/1ON95440XPDYfDCofD2RRtwBJ70wT9dI0AAGBDVi1wKBTSzJkz1djYmDzmOI4aGxtVVVV1weueeOIJrVq1Stu2bdOsWbMGXtociCZ6RsgiAABYkVXPiCTV1tZq4cKFmjVrlmbPnq1169apo6NDixYtkiQtWLBAEydOVH19vSTpn//5n7Vy5Uo9//zzqqioSI4tGTVqlEaNGjWIb2VgHAawAgBgVdZhZN68eTpx4oRWrlyp5uZmzZgxQ9u2bUsOaj18+LD8ad0MP/7xj9Xd3a0vf/nLGfepq6vTd77znUsr/SBIPKZhnREAAOzIep0RG3K1zojjGE16pEGStOvRal02yp1xKgAAeEFO1hkZbmJpOYzHNAAA2OHtMOKkwoifMAIAgBWeDiNOes8IY0YAALDC02EkvWeExzQAANhBGIkjjAAAYAdhJI7HNAAA2OHtMGIYwAoAgG2eDiOO0/s9SBABAMAaT4eRaDyN0CsCAIA9ng4jiZ4RxosAAGCPp8NIcl8aekYAALDG22GEHXsBALCOMCLCCAAANhFGJPkZMwIAgDWeDiNOcsyI5YIAAOBhnm6GEz0jQb+nqwEAAKs83QpHE49pPF0LAADY5elmOPmYhjEjAABY4+kwkhzAymwaAACs8XQYcZJjRggjAADY4ukwEmVqLwAA1nk6jLAcPAAA9nk6jDiswAoAgHWeDiMsBw8AgH2EETG1FwAAm7wdRgxTewEAsM3bYYSeEQAArPN0GEmswBoMEEYAALDF02EkGmOdEQAAbPN0GHFYZwQAAOs8HUZiTu93ekYAALDH22HEsDcNAAC2eTuMxLtGeEwDAIA93g4jvR0jrDMCAIBFng4jyb1pyCIAAFjj6TCS2rXX09UAAIBVnm6FUxvlWS4IAAAe5ulmmF17AQCwjzAi1hkBAMAmT4cRh3VGAACwztNhJJroGSGMAABgjafDSGpqL2EEAABbPB1GGMAKAIB93g4j7NoLAIB13g4j9IwAAGAdYURM7QUAwCZPhxGHxzQAAFjn6TDCYxoAAOzzdBiJEkYAALDO02GEdUYAALDP02Ek1ptFWIEVAACLPB1GEj0j7E0DAIA9ng4jUceRRM8IAAA2eTqMxHqzCGNGAACwyNNhJLXOiOWCAADgYZ5uhlPrjHi6GgAAsGpArfD69etVUVGh/Px8VVZWaufOnRc9/9e//rWmTJmi/Px83XDDDWpoaBhQYQdbKoxYLggAAB6WdTO8ZcsW1dbWqq6uTrt379b06dNVU1Oj48eP93n+jh07dPfdd+vee+/Vnj17dNddd+muu+7SX/7yl0su/KVibxoAAOzLOoysXbtWixcv1qJFi3Tddddpw4YNKigo0KZNm/o8/wc/+IE+//nP61vf+pamTp2qVatW6aabbtLTTz99yYW/VDH2pgEAwLqswkh3d7d27dql6urq1A38flVXV6upqanPa5qamjLOl6SampoLni9JXV1dikQiGV+5wDojAADYl1UYaW1tVSwWU2lpacbx0tJSNTc393lNc3NzVudLUn19vYqKipJf5eXl2RSz36I8pgEAwLqP5dDN5cuXq62tLfl15MiRnLzOl2deoQf/brImXT4yJ/cHAAAfLZjNycXFxQoEAmppack43tLSorKysj6vKSsry+p8SQqHwwqHw9kUbUD++3/5RM5fAwAAXFxWPSOhUEgzZ85UY2Nj8pjjOGpsbFRVVVWf11RVVWWcL0mvvvrqBc8HAADeklXPiCTV1tZq4cKFmjVrlmbPnq1169apo6NDixYtkiQtWLBAEydOVH19vSTpoYce0m233aannnpKd955pzZv3qzXX39dzzzzzOC+EwAAMCRlHUbmzZunEydOaOXKlWpubtaMGTO0bdu25CDVw4cPy5+2ouktt9yi559/Xo8++qgeeeQRffKTn9TLL7+sadOmDd67AAAAQ5bPmPhiGx9jkUhERUVFamtrU2Fhoe3iAACAfuhv+/2xnE0DAAC8gzACAACsIowAAACrCCMAAMAqwggAALCKMAIAAKwijAAAAKsIIwAAwCrCCAAAsCrr5eBtSCwSG4lELJcEAAD0V6Ld/qjF3odEGGlvb5cklZeXWy4JAADIVnt7u4qKii749yGxN43jODp27JhGjx4tn883aPeNRCIqLy/XkSNH2PMmh6hn91DX7qCe3UE9uyOX9WyMUXt7uyZMmJCxie65hkTPiN/v1xVXXJGz+xcWFvIP3QXUs3uoa3dQz+6gnt2Rq3q+WI9IAgNYAQCAVYQRAABglafDSDgcVl1dncLhsO2iDGvUs3uoa3dQz+6gnt3xcajnITGAFQAADF+e7hkBAAD2EUYAAIBVhBEAAGAVYQQAAFg17MPI+vXrVVFRofz8fFVWVmrnzp0XPf/Xv/61pkyZovz8fN1www1qaGhwqaRDWzb1vHHjRs2ZM0djx47V2LFjVV1d/ZH/uyAl23/TCZs3b5bP59Ndd92V2wIOE9nW86lTp7RkyRKNHz9e4XBY11xzDf/96Ids63ndunW69tprNWLECJWXl2vp0qU6e/asS6Udmv74xz9q7ty5mjBhgnw+n15++eWPvGb79u266aabFA6HdfXVV+u5557LbSHNMLZ582YTCoXMpk2bzF//+lezePFiM2bMGNPS0tLn+X/6059MIBAwTzzxhHnzzTfNo48+avLy8swbb7zhcsmHlmzr+Z577jHr1683e/bsMfv27TN///d/b4qKiszf/vY3l0s+9GRb1wmHDh0yEydONHPmzDFf+tKX3CnsEJZtPXd1dZlZs2aZO+64w7z22mvm0KFDZvv27Wbv3r0ul3xoybaef/GLX5hwOGx+8YtfmEOHDplXXnnFjB8/3ixdutTlkg8tDQ0NZsWKFebFF180ksxLL7100fMPHjxoCgoKTG1trXnzzTfND3/4QxMIBMy2bdtyVsZhHUZmz55tlixZkvw9FouZCRMmmPr6+j7P/8pXvmLuvPPOjGOVlZXmH/7hH3JazqEu23o+VzQaNaNHjzY/+9nPclXEYWMgdR2NRs0tt9xifvKTn5iFCxcSRvoh23r+8Y9/bCZNmmS6u7vdKuKwkG09L1myxHzmM5/JOFZbW2tuvfXWnJZzOOlPGPn2t79trr/++oxj8+bNMzU1NTkr17B9TNPd3a1du3apuro6eczv96u6ulpNTU19XtPU1JRxviTV1NRc8HwMrJ7P1dnZqZ6eHo0bNy5XxRwWBlrX3/3ud1VSUqJ7773XjWIOeQOp59/85jeqqqrSkiVLVFpaqmnTpmn16tWKxWJuFXvIGUg933LLLdq1a1fyUc7BgwfV0NCgO+64w5Uye4WNtnBIbJQ3EK2trYrFYiotLc04XlpaqrfeeqvPa5qbm/s8v7m5OWflHOoGUs/nevjhhzVhwoTz/vEj00Dq+rXXXtOzzz6rvXv3ulDC4WEg9Xzw4EH9/ve/11e/+lU1NDTowIEDevDBB9XT06O6ujo3ij3kDKSe77nnHrW2turTn/60jDGKRqN64IEH9Mgjj7hRZM+4UFsYiUR05swZjRgxYtBfc9j2jGBoWLNmjTZv3qyXXnpJ+fn5toszrLS3t2v+/PnauHGjiouLbRdnWHMcRyUlJXrmmWc0c+ZMzZs3TytWrNCGDRtsF21Y2b59u1avXq0f/ehH2r17t1588UVt3bpVq1atsl00XKJh2zNSXFysQCCglpaWjOMtLS0qKyvr85qysrKszsfA6jnhySef1Jo1a/S73/1ON954Yy6LOSxkW9fvvPOO3n33Xc2dOzd5zHEcSVIwGNT+/fs1efLk3BZ6CBrIv+nx48crLy9PgUAgeWzq1Klqbm5Wd3e3QqFQTss8FA2knh977DHNnz9f9913nyTphhtuUEdHh+6//36tWLFCfj+frwfDhdrCwsLCnPSKSMO4ZyQUCmnmzJlqbGxMHnMcR42NjaqqqurzmqqqqozzJenVV1+94PkYWD1L0hNPPKFVq1Zp27ZtmjVrlhtFHfKyrespU6bojTfe0N69e5NfX/ziF3X77bdr7969Ki8vd7P4Q8ZA/k3feuutOnDgQDLsSdLbb7+t8ePHE0QuYCD13NnZeV7gSARAwzZrg8ZKW5izobEfA5s3bzbhcNg899xz5s033zT333+/GTNmjGlubjbGGDN//nyzbNmy5Pl/+tOfTDAYNE8++aTZt2+fqaurY2pvP2Rbz2vWrDGhUMi88MIL5v33309+tbe323oLQ0a2dX0uZtP0T7b1fPjwYTN69GjzjW98w+zfv9/89re/NSUlJeZ73/uerbcwJGRbz3V1dWb06NHml7/8pTl48KD5t3/7NzN58mTzla98xdZbGBLa29vNnj17zJ49e4wks3btWrNnzx7z3nvvGWOMWbZsmZk/f37y/MTU3m9961tm3759Zv369UztvVQ//OEPzZVXXmlCoZCZPXu2+fOf/5z822233WYWLlyYcf6vfvUrc80115hQKGSuv/56s3XrVpdLPDRlU8+f+MQnjKTzvurq6twv+BCU7b/pdISR/su2nnfs2GEqKytNOBw2kyZNMt///vdNNBp1udRDTzb13NPTY77zne+YyZMnm/z8fFNeXm4efPBB8+GHH7pf8CHkD3/4Q5//zU3U7cKFC81tt9123jUzZswwoVDITJo0yfz0pz/NaRl9xtC3BQAA7Bm2Y0YAAMDQQBgBAABWEUYAAIBVhBEAAGAVYQQAAFhFGAEAAFYRRgAAgFWEEQAAYBVhBAAAWEUYAQAAVhFGAACAVYQRAABg1f8H1+E7VoFDc40AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr,tpr, thresholds = roc_curve(np.concatenate((np.zeros_like(cifar_softmax),np.ones_like(mnist_softmax))),np.concatenate(( cifar_softmax, mnist_softmax)))\n",
    "plt.plot(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.987191325\n"
     ]
    }
   ],
   "source": [
    "print('AUC:', auc(fpr,tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ood",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}