investigate_eval_fxn_errors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import sys,os\n",
    "\n",
    "from torchvision import transforms\n",
    "\n",
    "from tqdm import tqdm\n",
    "import pdb\n",
    "\n",
    "import pipeline.criterion as Criterion\n",
    "from pipeline.load import get_dataloader\n",
    "import pipeline.network as Network\n",
    "import LovaszSoftmax.pytorch.lovasz_losses as L\n",
    "\n",
    "import argparse\n",
    "import evaluate as Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ObjectView:\n",
    "    '''\n",
    "    Helper class to access dict values as attributes.\n",
    "\n",
    "    Replaces command-line arg-parse options.\n",
    "    '''\n",
    "    def __init__(self, d):\n",
    "        self.__dict__ = d\n",
    "\n",
    "\n",
    "def img_frombytes(data):\n",
    "    size = data.shape[::-1]\n",
    "    databytes = np.packbits(data, axis=1)\n",
    "    return Image.frombytes(mode='1', size=size, data=databytes)\n",
    "\n",
    "def load_model_with_weights(model_name=None, num_epochs=8, batch_size=16, use_lovasz=True, se_loss=False, aux=False):\n",
    "    '''\n",
    "    Load a model by name from the /models subdirectory.\n",
    "    '''\n",
    "\n",
    "    options = {\n",
    "        'use_jaccard': True,\n",
    "        'use_lovasz': use_lovasz,\n",
    "        'early_stopping': False,\n",
    "        'validation': True,\n",
    "        'model': 'encnet', # model name (default: encnet)\n",
    "        'backbone': 'resnet50', # backbone name (default: resnet50)\n",
    "        'jpu': True, # 'JPU'\n",
    "        'dilated': True, # 'dilation'\n",
    "        'lateral': False, #'employ FPN')\n",
    "        'dataset':'ade20k', # 'dataset name (default: pascal12)')\n",
    "        'workers': 16, # dataloader threads\n",
    "        'base_size': 520, # 'base image size'\n",
    "        'crop_size': 480, # 'crop image size')\n",
    "        'train_split':'train', # 'dataset train split (default: train)'\n",
    "\n",
    "        # training hyper params\n",
    "        'aux': aux, # 'Auxilary Loss'\n",
    "        'aux_weight': 0.2, # 'Auxilary loss weight (default: 0.2)'\n",
    "        'se_loss': se_loss, # 'Semantic Encoding Loss SE-loss'\n",
    "        'se_weight': 0.2, # 'SE-loss weight (default: 0.2)'\n",
    "        'start_epoch': 0, # 'start epochs (default:0)'\n",
    "        'batch_size': batch_size, # 'input batch size for training (default: auto)'\n",
    "        'test_batch_size': None, # 'input batch size for testing (default: same as batch size)'\n",
    "        'epochs':num_epochs,\n",
    "\n",
    "        # optimizer params\n",
    "        'optimizer': 'sgd',\n",
    "        'lovasz_hinge': True,\n",
    "        'lr': None, # 'learning rate (default: auto)'\n",
    "        'lr_scheduler': 'poly', # 'learning rate scheduler (default: poly)'\n",
    "        'momentum': 0.9, # 'momentum (default: 0.9)'\n",
    "        'weight_decay': 1e-4, # 'w-decay (default: 1e-4)'\n",
    "\n",
    "        # cuda, seed and logging\n",
    "        'no_cuda': False, # 'disables CUDA training'\n",
    "        'seed': 100, # 'random seed (default: 1)'\n",
    "\n",
    "        # checking point\n",
    "        'resume': None, # 'put the path to resuming file if needed'\n",
    "        'checkname': 'default', # 'set the checkpoint name'\n",
    "        'model-zoo': None, # 'evaluating on model zoo model'\n",
    "\n",
    "        # finetuning pre-trained models\n",
    "        'ft': False, # 'finetuning on a different dataset'\n",
    "\n",
    "        # evaluation option\n",
    "        'split': 'val',\n",
    "        'mode': 'testval',\n",
    "        'ms': False, # 'multi scale & flip'\n",
    "        'no_val': False, # 'skip validation during training'\n",
    "    }\n",
    "\n",
    "    model_args = ObjectView(options)\n",
    "    model = Network.get_model(model_args)\n",
    "\n",
    "    if model_name[-5:] == '_m.pt':\n",
    "        model_name = model_name[:-5]\n",
    "\n",
    "    path_to_state_dict = 'models/{}/{}_m.pt'.format(model_name, model_name)\n",
    "    model.load_state_dict(torch.load(path_to_state_dict))\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def output_to_pred_imgs(output, dim=0, use_lovasz=False):\n",
    "    \n",
    "    if use_lovasz:\n",
    "        np_pred = (output[0]>0).squeeze().cpu()\n",
    "    else:\n",
    "        np_pred = torch.max(output, dim=dim)[1].cpu().numpy()\n",
    "    return img_frombytes(np_pred)\n",
    "\n",
    "\n",
    "def predict_test_set(model, model_name, overwrite=False, use_lovasz=False):\n",
    "    '''\n",
    "    Predict for the entire submission set.\n",
    "    '''\n",
    "\n",
    "    out_dir = 'models/{}/predictions'.format(model_name)\n",
    "    if not os.path.exists(out_dir):\n",
    "        os.makedirs(out_dir)\n",
    "\n",
    "    test_dataloader = get_dataloader(load_test=True, batch_size=4, overwrite=overwrite, out_dir=out_dir)\n",
    "    if test_dataloader == False:\n",
    "        print('Exiting...')\n",
    "        sys.exit()\n",
    "    \n",
    "    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "    model.to(device)\n",
    "\n",
    "    print('Beginning Prediction Loop')\n",
    "    tbar = tqdm(test_dataloader)\n",
    "    for _, (image_tensors, img_names) in enumerate(tbar):        \n",
    "\n",
    "        # Load tensors to GPU\n",
    "        image_tensors = image_tensors.to(device)\n",
    "\n",
    "        # Predict on image tensors\n",
    "        with torch.no_grad():\n",
    "            outputs = model(image_tensors)\n",
    "            predict_imgs = [output_to_pred_imgs(output,use_lovasz=use_lovasz) for output in outputs]\n",
    "            \n",
    "        # Zip images, and save.\n",
    "        for predict_img, img_name in zip(predict_imgs, img_names):\n",
    "            image_out_path = 'models/{}/predictions/{}.tif'.format(model_name, img_name)\n",
    "            predict_img.save(image_out_path, compression=\"tiff_deflate\")\n",
    "\n",
    "    return None\n",
    "\n",
    "\n",
    "def score_region(model, model_name, region, thresh=0):\n",
    "    '''\n",
    "    evaluate and score for a single region. save results.\n",
    "    '''\n",
    "\n",
    "    test_dataloader = get_dataloader(\n",
    "        in_dir='training_data', batch_size=8, region=region\n",
    "        )\n",
    "    device = torch.device('cpu')\n",
    "    model.to(device)\n",
    "\n",
    "    print('Beginning Prediction Loop')\n",
    "\n",
    "    region_loss = 0\n",
    "\n",
    "    region_ct = 0\n",
    "    tbar = tqdm(test_dataloader)\n",
    "    with open('region_scores.csv', 'a', newline='') as csvfile:\n",
    "        csvwriter=csv.DictWriter(csvfile, fieldnames=['region', 'img_name', 'iou_score'])\n",
    "        csvwriter.writeheader()\n",
    "        for _, (images, masks, img_names) in enumerate(tbar):\n",
    "\n",
    "            # Predict on image tensors\n",
    "            with torch.no_grad():\n",
    "                images = images.to(device)\n",
    "\n",
    "                outputs = model(images)\n",
    "                outputs = (outputs[0]>thresh).long().data\n",
    "                masks = masks.to(device)\n",
    "\n",
    "                for i, img_name in enumerate(np.array(img_names)):\n",
    "                    loss = L.iou_binary(outputs[i], masks[i])\n",
    "                    region_loss += loss\n",
    "                    region_ct += 1\n",
    "                    img_name = img_name.split('/')[-1]\n",
    "                    csvwriter.writerow({'region':region, 'img_name':img_name, 'iou_score': loss})\n",
    "        \n",
    "        csvwriter.writerow({'region':region, 'img_name': 'avg_loss', 'iou_score': region_loss/region_ct})\n",
    "\n",
    "\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = '11-03-2020_12-11__full_lovasz_trial_chkpt'\n",
    "MODEL = load_model_with_weights(model_name=MODEL_NAME, use_lovasz=True)\n",
    "MODEL.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/2871 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting dataset.\n",
      "Pulling from directory training_data\n",
      "Loading Test\n",
      "Beginning Prediction Loop\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 6/2871 [00:06<53:51,  1.13s/it]  "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-3312820e3dc2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpredict_test_set\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMODEL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMODEL_NAME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_lovasz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-23-97527e5a7dd5>\u001b[0m in \u001b[0;36mpredict_test_set\u001b[0;34m(model, model_name, overwrite, use_lovasz)\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m             \u001b[0mpredict_imgs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0moutput_to_pred_imgs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_lovasz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_lovasz\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;32min\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0;31m# Zip images, and save.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-97527e5a7dd5>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m             \u001b[0mpredict_imgs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0moutput_to_pred_imgs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_lovasz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_lovasz\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;32min\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0;31m# Zip images, and save.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-97527e5a7dd5>\u001b[0m in \u001b[0;36moutput_to_pred_imgs\u001b[0;34m(output, dim, use_lovasz)\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0muse_lovasz\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m         \u001b[0mnp_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0mnp_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "predict_test_set(model=MODEL, model_name=MODEL_NAME, use_lovasz=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "787"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = glob.glob('models/11-03-2020_12-11__full_lovasz_trial_chkpt/predictions/*')\n",
    "len(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABAAAAAQAAQAAAABXZhYuAAAJxElEQVR4nO2dTXLjuBWAnyxV3FOVirScxVRZR5hldtZRkltkFTM5QR/BR+Gkssiyj8CZVS/ZVb1wu2wzC0sWZYHED/Hw1Ozv24imSODzIwmCEAGIjLLo3njprb7t3vMiImcru66rxtMXuRr/ujsuPo6b+jJKFJDd29Jnb1qVhkBzWPhtm5L8dIH2sFDr5O8VeKMyEnjYf3ajW0m6YGgEXvyb6Ag8aWUcKnDgVKQJTr/KJTA9p0SBw7F/GN3qlVpDwE2btFdGgYyECrTWAj68BVWqgDvhkFMyk8CB+3xZRnJ3Vh0SkaWzRnReT+r8R8YbgY8i4qsOTcEr8GXx98WHbx9OV4bfmqrJAiL38j7/nKRdhskXXS6BjCQKVNYC+UgUqK0F8pEo0FgLtO9XqD0b6rHouiZdIMP9+C8iv6QLBLMb/OZXkWVXpwoEP6/Ug9/sRES2hudA9fqRKOC8HzcxKSxFRGRjFIHrrns9iB9SBZz34zZ4938cF00isKgOS6tkgcq7xQhXA8tTCS6dVr3sUwXqxP3OyBmB4NKpv2GqQJO4XzZuHI9A589LA09Gxw2fkiPQOtYltaSlCky6H/dNlQuiDA+n2qQKuC65pAc25Qi4z8ur3pKygK9sWiQLOP+1KnDnVW9ZOQLeq7VSbitunWv77R2pApOaKDYZBLKRVaAO3G6rJZCCtUC+q0AkvJay6y0bR+A5r0AbuF31tpReIwpsEdl4vv+cLwK7pL2avIcgsJ7Wi16dnNfiffW3FpHrs0qxW6lfac4agejfeXMLnLMa//pFXcCffFaB0CeT6rDwZF0SPqgL+MsrkwjUh4U2s0BoPa3pLZvUCdueickheCsfayOBLG/GuO4FjvfpKte+bzcSmRABZ8qBdL1Pm4Koev14ySlQ9xIO4Xd5PRdsIlDL83Yj8r+cAlXc5l/ly7bb5RSI5JPI71eSUSCuZGyOtwOrCFS5BeJaSdtjwHIJRJauR1/tQ1A51z58zS4Q13b8tJme4+3pTacREdfNyJtOrgjcZ0onnBvXf2oXgfQaWiaB9NeOkwXak7++ujfSFDi97j6VFzgt+upkgXSO1dHj+44JV0E6d/ssXuS6O4SjqMChIGhFbioLgfVrDicnY4JA8htQsnwSEXk+aYQ5z8+bvqcRZ4SXoPS9pJeEXSXybXL+U7h5/7ax4xzwF9HTY3gicLbCG+G8VbIqfheTn24LCviZu4D/cWHuEfB3UlQWuDcW8Hc5VRYI6JihK+A/BZQFGmuBWjV1B7fxNULVCJgLhDwxqgqENBypCoS026gKfLIT+I+IWDyyHi7DVq6DqsSiFYGP8m0l8odK2qPc9p4X1+qdth3sn5hrg6z7Ahb/+p51V8u6sssfAAAAAAAAAAAAAAAAAAAAAAAAAGCUgN4WeiyHO76V6ebzi0jufnVxvL5sW7m+KhKB1yHC3YO3FRHYv+e7tRG4e+vpsVHPy0VvuDhndwP1CPx8XHT28FUX2HryUhfYHRdtCoJ+t6PK8b12BLz/tbbASfo7A4E/+zbQFmiNBe50k/fnf9r5ri6c/fvOf/sRc4pxPmioS0DxHPirXtJhnAWgcARuwjZTE7hubAUWoYNUaQkED1CkJLAO3lJJ4FdrgZ1OsuGclwFly4GI2p+OQESqOgIRY/zoCETMCDnTYUA21gIR6AhsrQUimKnAzlrAHPe9qHS1PAwE5ikQ0xo1zwggMMSmlEBMoioCMeNOqgjEzFKuIrCxFthaC+wG1rsOjUob/tBoXK4p7GdZEEVFtWgEXOWDhkBUmkUj4MpsliehucDgvch1ecwyAuYCw8OCVWUEotAQiBrKf5YRMBeIGi9+lhGwF6isBWIoK7ArJFBHbPuDHYJSAo21QAwIIKAi0FoLxIAAAgioCMTMtDfPCFyuwNZawAECCCCAwDwFYmZQKBuBTSGBmKbSeZ4DMS2V84zAID/KT7cIXPCbVI5fFOd5CBBAgJsRApcr4KguzzQC1cB6R/PZTCMwRGst4KCsQGMt4GCmAnX4+plGwJybgX4+jk3NI1BU4AeKQOte7Wo7KhoBV/NdUQHXLORFBRprgdqxTmeMoGv3T5fW7xU7n9h0BNyt1c4W3JIR+Got8Mm1UuckXDqPgTMvnQg4j7a71aDgIbAcn1BEhi6NggLuwklHwHm424ICThprgbpcVmfDcXXd0CCdxSLwr2XZ2uftyb/ede3glgW8/i1b+aifzSm9CLQi11G9TrJwfDbzvdalfghcNeECrMeeR0uwCDwAeqy77kGW5U++I6GDAaqdhCu510o6kH8a5+8fnxMAAAAAAAAAAEb4m3H+tyXn+3D9mtmVnPnH0U64EJHyk570WAY3tedgtKX0utIXcBzs3vsXnf4vOuM5LPSnnxmPgMhzzKirKfhivKyUBRy8m4WnsRYY+9UxAyGnuWrDe4jAckL66yp+n/OZmBISOXDn64EbVNLs0gVSfpQKm4sqEO8vl0GCm3QBL0ECMUNAn1FlEJhUHO8yCEy6J24V0w5ik0FgUg3RcwJpR6DybaAtUPuG5tEWaHxjhGkLtL6h+rQFHhJeITu/F0x5E8U3yaSvUioi02rnK8/9WL0gylIf0MQhEDOKiIpAWeYv4KtLqAv4MlAX+CmHwKR30uoMAlPYbI0FthtjgZ2nSua4GZ13j5jSTNJ5dp59QWQ+7fifRIb6vJQR+FlEZDWSi7bAZ7kSeRopybQFHkXGryJtga6Kf65bnlWKp9SRbjxVavOX29UFHj3Pp+oCWRqpJlHtRr9Wbo1fPUkdfRnkvApW3vm3lQ/BlcgXUwE/cxfwn4Bzj4C5gL9xJ0ggvRx4zCOgiUOgbO+kS4wAAgiU7aN4iRFA4AIFNF9o+j4igEBrLaDJ9yHQWAto8n0I1NYCmiDgEqisBYqCAAJcBQggYC4w0MvGv1EuzCOAAAIIIIAAAgh8HwKVtYAmCCCAAAIIXKJAufEnBgTKgsAlCpR1usQImAtoj7vhFSgLAg6BSaMN5BAwZ332brlmbo4IbDTzCxEoi0Ngay3QWAuUxSHQ7j/rghon7Lv/P91YXYYi8m0r8t9WM98xll1XXXcvx4EQSs9Esew6WXbtsbdNaYFF9yLSSSEB5znwKLIQedbMd0ygq/Y95Kr9CrvZSNbTuxtOYz8hjOoogaNFcbf4IGI2Ic/ewfIIvBrYTcgDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8I/wcR9t0NFDtWfQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.TiffImagePlugin.TiffImageFile image mode=1 size=1024x1024 at 0x7FD7B3740310>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img = Image.open(outputs[700])\n",
    "img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Image.getbands of <PIL.TiffImagePlugin.TiffImageFile image mode=1 size=1024x1024 at 0x7FE8288E6F90>>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.getbands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fe82894c790>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ8AAAD8CAYAAABpXiE9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOdklEQVR4nO3cb6yedX3H8fdn/YfgsC3+CbbNCrFxM0s22IkUXYyx6qQzlgeQYMzsWJcmm9tUlmjZHphtT3Qx4kwWtLG6ujiEVTIawmawYJY9sKOoQ6Bij7jRIygY/mg0wxK/e3D/CjftAdrffc5932e+X8nJfV3f63ed63t+HD69ruu+z5WqQpJO1y9NugFJS5PhIamL4SGpi+EhqYvhIamL4SGpy9jDI8nbktyXZDbJrnEfX9LCyDg/55FkGfBt4C3AHHAH8M6qundsTUhaEOM+83gtMFtV91fVz4AvANvG3IOkBbB8zMdbBxwdWp8DLhoekGQnsBNgGct+60zOHl930i+gH/PYD6vqZae737jDI/PUnnXdVFW7gd0AZ2dtXZQt4+hL+oX15dr3Pz37jfuyZQ7YMLS+HnhwzD1IWgDjDo87gE1JzkuyErgC2D/mHiQtgLFetlTVU0n+BPgSsAz4TFXdM84eJC2Mcd/zoKpuAW4Z93ElLSw/YSqpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpS3d4JNmQ5PYkh5Pck+S9rb42ya1JjrTXNa2eJJ9IMpvkriQXLtQPIWn8RjnzeAr486r6NWAz8J4krwF2AQeqahNwoK0DXAJsal87gWtHOLakCesOj6p6qKq+1pZ/DBwG1gHbgL1t2F7g0ra8DfhcDXwVWJ3k3O7OJU3UgtzzSLIRuAA4CLyiqh6CQcAAL2/D1gFHh3aba7UTv9fOJIeSHDrGkwvRnqRFMHJ4JHkx8EXgfVX1o+cbOk+tTipU7a6qmaqaWcGqUduTtEhGCo8kKxgEx+er6sZW/sHxy5H2+nCrzwEbhnZfDzw4yvElTc4o77YE2AMcrqqPDW3aD2xvy9uBm4bq727vumwGnjh+eSNp6Vk+wr6vB34P+GaSb7TaXwAfBm5IsgN4ALi8bbsF2ArMAj8Frhzh2JImrDs8quo/mP8+BsCWecYX8J7e40maLn7CVFIXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUpeRwyPJsiRfT3JzWz8vycEkR5Jcn2Rlq69q67Nt+8ZRjy1pchbizOO9wOGh9Y8A11TVJuAxYEer7wAeq6pXAde0cZKWqJHCI8l64HeBT7f1AG8C9rUhe4FL2/K2tk7bvqWNl7QEjXrm8XHgA8DP2/o5wONV9VRbnwPWteV1wFGAtv2JNv5ZkuxMcijJoWM8OWJ7khZLd3gkeTvwcFXdOVyeZ2idwrZnClW7q2qmqmZWsKq3PUmLbPkI+74eeEeSrcAZwNkMzkRWJ1nezi7WAw+28XPABmAuyXLgJcCjIxxf0gR1n3lU1dVVtb6qNgJXALdV1buA24HL2rDtwE1teX9bp22/rapOOvOQtDQsxuc8PghclWSWwT2NPa2+Bzin1a8Cdi3CsSWNySiXLU+rqq8AX2nL9wOvnWfM/wKXL8TxJE2enzCV1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1MXwkNTF8JDUxfCQ1GWk8EiyOsm+JN9KcjjJxUnWJrk1yZH2uqaNTZJPJJlNcleSCxfmR5A0CaOeefwd8G9V9avAbwCHgV3AgaraBBxo6wCXAJva107g2hGPLWmCusMjydnAG4A9AFX1s6p6HNgG7G3D9gKXtuVtwOdq4KvA6iTndncuaaJGOfM4H3gE+GySryf5dJKzgFdU1UMA7fXlbfw64OjQ/nOt9ixJdiY5lOTQMZ4coT1Ji2mU8FgOXAhcW1UXAD/hmUuU+WSeWp1UqNpdVTNVNbOCVSO0J2kxjRIec8BcVR1s6/sYhMkPjl+OtNeHh8ZvGNp/PfDgCMeXNEHd4VFV3weOJnl1K20B7gX2A9tbbTtwU1veD7y7veuyGXji+OWNpKVn+Yj7/ynw+SQrgfuBKxkE0g1JdgAPAJe3sbcAW4FZ4KdtrKQlaqTwqKpvADPzbNoyz9gC3jPK8SRNDz9hKqmL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqYvhIamL4SGpi+EhqctI4ZHk/UnuSXJ3kuuSnJHkvCQHkxxJcn2SlW3sqrY+27ZvXIgfQNJkdIdHknXAnwEzVfXrwDLgCuAjwDVVtQl4DNjRdtkBPFZVrwKuaeMkLVGjXrYsB16UZDlwJvAQ8CZgX9u+F7i0LW9r67TtW5JkxONLmpDu8Kiq7wEfBR5gEBpPAHcCj1fVU23YHLCuLa8DjrZ9n2rjzznx+ybZmeRQkkPHeLK3PUmLbJTLljUMzibOA14JnAVcMs/QOr7L82x7plC1u6pmqmpmBat625O0yEa5bHkz8N2qeqSqjgE3Aq8DVrfLGID1wINteQ7YANC2vwR4dITjS5qgUcLjAWBzkjPbvYstwL3A7cBlbcx24Ka2vL+t07bfVlUnnXlIWhpGuedxkMGNz68B32zfazfwQeCqJLMM7mnsabvsAc5p9auAXSP0LWnCMs3/+J+dtXVRtky6Den/tS/XvjurauZ09/MTppK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6GB6SuhgekroYHpK6vGB4JPlMkoeT3D1UW5vk1iRH2uuaVk+STySZTXJXkguH9tnexh9Jsn1xfhxJ43IqZx7/ALzthNou4EBVbQIOtHWAS4BN7WsncC0Mwgb4EHAR8FrgQ8cDR9LS9ILhUVX/Djx6QnkbsLct7wUuHap/rga+CqxOci7wO8CtVfVoVT0G3MrJgSRpCem95/GKqnoIoL2+vNXXAUeHxs212nPVT5JkZ5JDSQ4d48nO9iQttoW+YZp5avU89ZOLVburaqaqZlawakGbk7RwesPjB+1yhPb6cKvPARuGxq0HHnyeuqQlqjc89gPH3zHZDtw0VH93e9dlM/BEu6z5EvDWJGvajdK3tpqkJWr5Cw1Ich3wRuClSeYYvGvyYeCGJDuAB4DL2/BbgK3ALPBT4EqAqno0yd8Ad7Rxf11VJ96ElbSEpGreWw9TIcmPgfsm3ccpeinww0k3cQqWSp+wdHpdKn3C/L3+SlW97HS/0QueeUzYfVU1M+kmTkWSQ0uh16XSJyydXpdKn7CwvfrxdEldDA9JXaY9PHZPuoHTsFR6XSp9wtLpdan0CQvY61TfMJU0vab9zEPSlDI8JHWZ2vBI8rYk97Vng+x64T0WtZcNSW5PcjjJPUne2+qn/VyTMfW7LMnXk9zc1s9LcrD1eX2Sla2+qq3Ptu0bx9zn6iT7knyrze3FUzyn72//7e9Ocl2SM6ZhXif6vJ2qmrovYBnwHeB8YCXwX8BrJtjPucCFbfmXgW8DrwH+FtjV6ruAj7TlrcC/MviDwM3AwTH3exXwT8DNbf0G4Iq2/Engj9ryHwOfbMtXANePuc+9wB+25ZXA6mmcUwZ/Af5d4EVD8/n70zCvwBuAC4G7h2qnNYfAWuD+9rqmLa95wWOP85flNCbkYuBLQ+tXA1dPuq+hfm4C3sLg06/nttq5DD7UBvAp4J1D458eN4be1jN4QNObgJvbL8oPgeUnzi2Dvy+6uC0vb+Mypj7Pbv9D5oT6NM7p8UdKrG3zdDODZ9RMxbwCG08Ij9OaQ+CdwKeG6s8a91xf03rZcsrP/xi3dgp6AXCQ03+uyTh8HPgA8PO2fg7weFU9NU8vT/fZtj/Rxo/D+cAjwGfbJdank5zFFM5pVX0P+CiDv+N6iME83cl0ziss4vN2hk1reJzy8z/GKcmLgS8C76uqHz3f0Hlqi95/krcDD1fVnafYyyTneTmD0+1rq+oC4Cc88zjL+Uys13bPYBtwHvBK4CwGj9x8rn6m8veXBXjezrBpDY+pe/5HkhUMguPzVXVjK5/uc00W2+uBdyT5b+ALDC5dPs7gcZDH/45puJen+2zbX8LJj5xcLHPAXFUdbOv7GITJtM0pwJuB71bVI1V1DLgReB3TOa8wpuftTGt43AFsanezVzK46bR/Us0kCbAHOFxVHxvadLrPNVlUVXV1Va2vqo0M5uy2qnoXcDtw2XP0ebz/y9r4sfwLWVXfB44meXUrbQHuZcrmtHkA2JzkzPa7cLzXqZvXeY6/eM/bGccNp86bQFsZvKvxHeAvJ9zLbzM4jbsL+Eb72srgOvYAcKS9rm3jA/x96/2bwMwEen4jz7zbcj7wnwyes/LPwKpWP6Otz7bt54+5x98EDrV5/RcGd/qnck6BvwK+BdwN/COwahrmFbiOwX2YYwzOIHb0zCHwB63fWeDKUzm2H0+X1GVaL1skTTnDQ1IXw0NSF8NDUhfDQ1IXw0NSF8NDUpf/A2GZ6G9oq8jNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(np.array(img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}