12_FRONT-parse_matches.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T16:43:52.905733Z",
     "start_time": "2019-10-30T16:43:51.531629Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# IMPORT\n",
    "%matplotlib inline\n",
    "import csv\n",
    "import os\n",
    "import time\n",
    "from operator import itemgetter, attrgetter, methodcaller\n",
    "from skimage.filters import threshold_otsu, threshold_adaptive, rank\n",
    "from skimage.morphology import disk\n",
    "from skimage.feature import match_template\n",
    "import matplotlib.image as img\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from PIL import Image, ImageDraw, ImageFont\n",
    "# Imports the Google Cloud client library\n",
    "from google.cloud import storage\n",
    "# Pandas is used for data manipulation\n",
    "import pandas as pd\n",
    "# for checking file existence\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T16:43:52.973709Z",
     "start_time": "2019-10-30T16:43:52.907864Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# READ crops\n",
    "crops = {}\n",
    "with open('crops.csv') as csvfile:\n",
    "    reader = csv.DictReader(csvfile, fieldnames=(\"file\",\"x_start\",\"y_start\",\"x_end\",\"y_end\",\"rotate\",\"horiz\",\"upside\"))\n",
    "    next(reader)\n",
    "    for row in reader:\n",
    "        row['name'] = row['file'].split(\"/\")[5] # stay with the folder name\n",
    "        row['name'] = row['name'].split(\"-\")\n",
    "        row['name'] = row['name'][0] + row['name'][1] # set the short name...\n",
    "        row['x_start'] = int(row['x_start'])\n",
    "        row['y_start'] = int(row['y_start'])\n",
    "        row['x_end'] = int(row['x_end'])\n",
    "        row['y_end'] = int(row['y_end'])\n",
    "        # import pdb; pdb.set_trace()\n",
    "        row['rotate'] = True if row['rotate'] == 'True' else False\n",
    "        crops[row['name']] = row\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T16:49:27.714284Z",
     "start_time": "2019-10-30T16:49:27.677136Z"
    },
    "code_folding": [
     51,
     65,
     107
    ]
   },
   "outputs": [],
   "source": [
    "# UTILITY func\n",
    "cube_size = 250\n",
    "HORIZ_TOLERANCE_FACTOR = 50\n",
    "VERT_TOLERANCE_FACTOR = 75\n",
    "EDGE_GAP = 50\n",
    "bucket_name = \"papyrus-thesis\"\n",
    "imgs_root = \"thesis-papyri/PAPYRI/\" # \"/Volumes/250GB/PAPYRI/\"\n",
    "cropped_root = \"/thesis-papyri/cropped/\" # \"/Volumes/250gb/cropped2\"\n",
    "matched_root = \"/thesis-papyri/non593_front/matched/\" # \"/Volumes/250gb/cropped2\"\n",
    "local_temp_root = \"/media/1KGB_ILAN/papyrus/files/temp/\"\n",
    "#local_temp_root = \"/Users/il239838/Downloads/temp/\"\n",
    "\n",
    "# Instantiates a client\n",
    "storage_client = storage.Client()\n",
    "\n",
    "def list_blobs_with_prefix(storage_client, prefix):\n",
    "    bucket = storage_client.get_bucket(bucket_name)\n",
    "    blobs = bucket.list_blobs(prefix=prefix, delimiter='/')\n",
    "\n",
    "    for blob in blobs:\n",
    "        if blob.name[-1] != '/':\n",
    "            yield blob.name\n",
    "        \n",
    "    #     if delimiter:\n",
    "    #         print('Prefixes:')\n",
    "    #         for prefix in blobs.prefixes:\n",
    "    #             print(prefix)\n",
    "\n",
    "def download_blob(storage_client, source_blob_name, destination_file_name):\n",
    "    bucket = storage_client.get_bucket(bucket_name)\n",
    "    blob = bucket.blob(source_blob_name)\n",
    "\n",
    "    blob.download_to_filename(destination_file_name)\n",
    "\n",
    "    # print('Blob {} downloaded to {}.'.format(\n",
    "    #    source_blob_name,\n",
    "    #    destination_file_name))\n",
    "\n",
    "def gcp_img_read(storage_client, chunk, img_path):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    fname = img_path[img_path.rfind(\"/\") + 1:]\n",
    "    local_fname = local_temp_root + str(chunk) + fname\n",
    "    download_blob(storage_client, img_path, local_fname)\n",
    "    while not Path(local_fname).is_file():\n",
    "        time.sleep(2)\n",
    "        print(\"Waiting for file: <<<\" + local_fname + \">>>\")\n",
    "    result = img.imread(local_fname)\n",
    "    time.sleep(2)\n",
    "    os.remove(local_fname)\n",
    "    return result\n",
    "\n",
    "def gcp_np_load(storage_client, img_path):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    fname = img_path[img_path.rfind(\"/\") + 1:]\n",
    "    local_fname = local_temp_root + fname\n",
    "    download_blob(storage_client, img_path, local_fname)\n",
    "    while not Path(local_fname).is_file():\n",
    "        time.sleep(2)\n",
    "        print(\"Waiting for file: <<<\" + local_fname + \">>>\")\n",
    "    result = np.load(local_fname)\n",
    "    time.sleep(2)\n",
    "    os.remove(local_fname)\n",
    "    return result\n",
    "    \n",
    "# Simple crop by x/y ranges\n",
    "def crop(image, ymin, ymax, xmin, xmax):\n",
    "    return image[ymin:ymax, xmin:xmax]\n",
    "\n",
    "\n",
    "def calc_combined_coordinates(base_x, base_y, offset_x, offset_y, base_rotate, base_x_end):\n",
    "    if base_rotate:\n",
    "        result_x = base_x_end - cube_size - offset_y # 250==CUBE_SIZE\n",
    "        result_y = base_y + offset_x\n",
    "    else:\n",
    "        result_x = base_x + offset_x\n",
    "        result_y = base_y + offset_y\n",
    "    return result_x, result_y\n",
    "\n",
    "\n",
    "# imgs_root = \"/Volumes/250GB/PAPYRI/\"\n",
    "# cropped_root = \"/Volumes/250gb/cropped2\"\n",
    "\n",
    "# IMPORTANT: kept the function name but it is actually pulling the front side!!!\n",
    "def load_img_for_name_FRONT(storage_client, chunk, file_name):\n",
    "    img_path = \"\"\n",
    "    if \"-\" in file_name:\n",
    "        name_parts = file_name.split(\"-\")\n",
    "        unique_part = name_parts[0] + \"-\" + name_parts[1] + \"-\" + name_parts[2]\n",
    "        img_path = imgs_root + name_parts[0] + \"/\" + \\\n",
    "            unique_part + \"/\"\n",
    "    else:\n",
    "        img_path = imgs_root + file_name[0:4] + \"/\" + \\\n",
    "            file_name[0:4] + \"-\" + file_name[4:9] + \"-R/\"\n",
    "    \n",
    "    for file_ in list_blobs_with_prefix(storage_client, img_path):\n",
    "        if (\" _018\" in file_ and '/._P' not in file_):        \n",
    "            return gcp_img_read(storage_client, chunk, file_) # img.imread(img_path + file_)\n",
    "\n",
    "def load_cropped_for_name(file_name):\n",
    "    img_path = cropped_root + \"/\" + file_name + \" _018.jpg.npy\"\n",
    "    return np.load(img_path)\n",
    "\n",
    "def load_cropped_for_name(file_name):\n",
    "    img_path = cropped_root + \"/\" + file_name + \" _018.jpg.npy\"\n",
    "    return gcp_np_load(img_path) # np.load(img_path)\n",
    "\n",
    "# IMPORTANT: kept the function name but it is actually pulling the back side!!!\n",
    "def load_front_for_name_FRONT(storage_client, chunk, file_name): \n",
    "    img_path = \"\"\n",
    "    if \"-\" in file_name:\n",
    "        front_file_name = file_name.replace(\"-R-\",\"-V-\")\n",
    "        name_parts = front_file_name.split(\"-\")\n",
    "        unique_part = name_parts[0] + \"-\" + name_parts[1] + \"-\" + name_parts[2]\n",
    "        img_path = imgs_root + name_parts[0] + \"/\" + \\\n",
    "            unique_part + \"/\"\n",
    "    else:\n",
    "        img_path = imgs_root + file_name[0:4] + \"/\" + \\\n",
    "            file_name[0:4] + \"-\" + file_name[4:9] + \"-V/\"\n",
    "    #import pdb; pdb.set_trace()\n",
    "    #for root, dirs, files in os.walk(img_path):\n",
    "    for file_ in list_blobs_with_prefix(storage_client, img_path):\n",
    "        if (\" _018\" in file_ and '/._P' not in file_):        \n",
    "            return gcp_img_read(storage_client, chunk, file_) # img.imread(img_path + file_)\n",
    "    \n",
    "# Pre-process the validation set\n",
    "# IMPORTANT: kept the function name but it is actually pulling the back side!!!\n",
    "def folder_walker_FRONT(storage_client, path, full_path, filter_text=\"\"):\n",
    "    result = []\n",
    "    #for root, dirs, files in os.walk(img_path):\n",
    "    for file_ in list_blobs_with_prefix(storage_client, path):\n",
    "        if \"-R-\" in file_ and not file_.startswith(\".\"):\n",
    "            if (filter_text == \"\" or filter_text in file_):\n",
    "                result.append(file_)\n",
    "    return result\n",
    "\n",
    "# no_rotate = folder_walker(storage.Client(), \"no_rotate/\", False)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-21T13:54:39.129655Z",
     "start_time": "2019-10-21T13:53:40.182012Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# CALC basic voting metrics per fragment and per side ALSO accumulate matches per fragment for trend (later)\n",
    "count = 0\n",
    "fragmentTotal = {} # Total number of fragments: usually indication of the size of the fragment\n",
    "fragmentVote = {} # basic fragment voting: one fragment vote per one match regardless of the side of the fragment\n",
    "fragmentAndSideTotal = {} # Total number of fragments per side: usually indication of the size of the side\n",
    "fragmentAndSideVote = {} # basic fragment-side voting: one fragment-side vote per one match on the specific side\n",
    "fragmentAndSideTrend = {}\n",
    "fragmentAndSideCubes = {}\n",
    "fragmentAndSideClass = {}\n",
    "origCoordinates = {}\n",
    "fragmentNames = {}\n",
    "matchFirstFile = {}\n",
    "matchSecondFile = {}\n",
    "firstNames = {}\n",
    "secondNames = {}\n",
    "fragmentAndSideDrawRect = {}\n",
    "fragmentAndSideMatchPoint = {}\n",
    "\n",
    "with open('20191021_FRONT_TRAINED_FRONT_pairs_match.csv') as csvfile:\n",
    "    # reader = csv.DictReader(csvfile, fieldnames=(\"firstName\",\"secondName\",\"gap\"))\n",
    "    reader = csv.DictReader(csvfile, fieldnames=(\"firstName\",\"secondName\",\"totalFragment\",\"totalSide\"))\n",
    "    for row in reader:\n",
    "        row['secondName'] = row['secondName'].split(\".\")[0] # get rid of the suffix\n",
    "\n",
    "        # split the first file name and calc the row and col\n",
    "        split = row['firstName'].split(\"_\")\n",
    "        row['firstNameOrig'] = split[0]\n",
    "        # import pdb; pdb.set_trace()\n",
    "        row['firstRow'] = int(int(split[1]) / 250)\n",
    "        row['firstY'] = int(split[1])\n",
    "        row['firstX'] = int(split[2])\n",
    "        row['firstCol'] = 0 if row['firstX'] < 100 else 1\n",
    "        split = row['firstNameOrig'].split(\"-\")\n",
    "        row['firstName'] = split[0] + split[1]\n",
    "\n",
    "        # split the second file name and calc the row and col\n",
    "        split = row['secondName'].split(\"_\")\n",
    "        row['secondNameOrig'] = split[0]\n",
    "        row['secondRow'] = int(int(split[1]) / 250)\n",
    "        row['secondY'] = int(split[1])\n",
    "        row['secondX'] = int(split[2])\n",
    "        row['secondCol'] = 0 if row['secondX'] < 100 else 1\n",
    "        split = row['secondNameOrig'].split(\"-\")\n",
    "        row['secondName'] = split[0] + split[1]\n",
    "\n",
    "        row['matchFragmentKey'] = row['firstName'] + \"_\" + row['secondName']\n",
    "        if row['matchFragmentKey'] not in fragmentTotal:\n",
    "            fragmentTotal[row['matchFragmentKey']] = 0\n",
    "            fragmentVote[row['matchFragmentKey']] = 0\n",
    "\n",
    "        fragmentVote[row['matchFragmentKey']] += 1\n",
    "        fragmentTotal[row['matchFragmentKey']] = \\\n",
    "            int(row['totalFragment']) if int(row['totalFragment']) > fragmentTotal[row['matchFragmentKey']] else fragmentTotal[row['matchFragmentKey']]\n",
    "\n",
    "        row['matchFragmentAndSideKey'] = row['firstName'] + \"_\" + str(row['firstCol']) \\\n",
    "            + \"_\" + row['secondName'] + \"_\" + str(row['secondCol'])\n",
    "        if row['matchFragmentAndSideKey'] not in fragmentAndSideVote:\n",
    "            fragmentAndSideClass[row['matchFragmentAndSideKey']] = 0\n",
    "            fragmentAndSideTotal[row['matchFragmentAndSideKey']] = 0\n",
    "            fragmentAndSideVote[row['matchFragmentAndSideKey']] = 0\n",
    "            fragmentAndSideTrend[row['matchFragmentAndSideKey']] = []\n",
    "            fragmentAndSideCubes[row['matchFragmentAndSideKey']] = []\n",
    "            origCoordinates[row['matchFragmentAndSideKey']] = []\n",
    "            fragmentNames[row['matchFragmentAndSideKey']] = row['matchFragmentKey']\n",
    "            firstNames[row['matchFragmentAndSideKey']] = row['firstName']\n",
    "            secondNames[row['matchFragmentAndSideKey']] = row['secondName']\n",
    "            fragmentAndSideDrawRect[row['matchFragmentAndSideKey']] = []\n",
    "            fragmentAndSideMatchPoint[row['matchFragmentAndSideKey']] = []\n",
    "            \n",
    "        fragmentAndSideTotal[row['matchFragmentAndSideKey']] = \\\n",
    "            int(row['totalSide']) if int(row['totalSide']) > fragmentAndSideTotal[row['matchFragmentAndSideKey']] else fragmentAndSideTotal[row['matchFragmentAndSideKey']]\n",
    "        fragmentAndSideVote[row['matchFragmentAndSideKey']] += 1\n",
    "\n",
    "        invert = True if row['firstCol'] == row['secondCol'] else False\n",
    "        fragmentAndSideTrend[row['matchFragmentAndSideKey']].append([invert, row['firstRow'], row['secondRow']])\n",
    "        fragmentAndSideCubes[row['matchFragmentAndSideKey']].append([row['firstX'], row['firstY'], row['secondX'], row['secondY']])\n",
    "\n",
    "        # TODO: FIXME:\n",
    "        # probably need to fix the next line since we moved to real matches to be able to handle flips - the x/y would not match\n",
    "        fragmentAndSideDrawRect[row['matchFragmentAndSideKey']].append([row['firstX'] + cube_size + EDGE_GAP - HORIZ_TOLERANCE_FACTOR, \n",
    "                                                                        row['firstY'] - row['secondY'] - VERT_TOLERANCE_FACTOR,\n",
    "                                                                        row['firstX'] + cube_size + EDGE_GAP + HORIZ_TOLERANCE_FACTOR, \n",
    "                                                                        row['firstY'] - row['secondY'] + VERT_TOLERANCE_FACTOR])\n",
    "        fragmentAndSideMatchPoint[row['matchFragmentAndSideKey']].append([row['firstX'] + cube_size + EDGE_GAP, \n",
    "                                                                          row['firstY'] - row['secondY']])\n",
    "\n",
    "        \n",
    "#         firstCrop = crops[row['firstName']]\n",
    "#         firstXcombined, firstYCombined = \\\n",
    "#             calc_combined_coordinates(firstCrop['x_start'], firstCrop['y_start'], row['firstX'], row['firstY'], firstCrop['rotate'], firstCrop['x_end'])\n",
    "#         secondCrop = crops[row['secondName']]\n",
    "#         secondXcombined, secondYCombined = \\\n",
    "#             calc_combined_coordinates(secondCrop['x_start'], secondCrop['y_start'], row['secondX'], row['secondY'], secondCrop['rotate'], secondCrop['x_end'])\n",
    "#         origCoordinates[row['matchFragmentAndSideKey']].append([firstXcombined, firstYCombined, secondXcombined, secondYCombined])\n",
    "#         matchFirstFile[row['matchFragmentAndSideKey']] = row['firstNameOrig']\n",
    "#         matchSecondFile[row['matchFragmentAndSideKey']] = row['secondNameOrig']\n",
    "\n",
    "#         # print(row['firstName'], row['firstRow'], row['firstCol'], row['secondName'], row['secondRow'], row['secondCol'], row['gap'])\n",
    "#         count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-21T13:54:42.115031Z",
     "start_time": "2019-10-21T13:54:39.131459Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# CALC simple trend voting\n",
    "fragmentAndSideTrendVote = {}\n",
    "for fragmanetAndSideKey in fragmentAndSideTrend:\n",
    "    # import pdb; pdb.set_trace()\n",
    "    fragmentAndSideTrend[fragmanetAndSideKey] = sorted(fragmentAndSideTrend[fragmanetAndSideKey], key=itemgetter(2), reverse=fragmentAndSideTrend[fragmanetAndSideKey][0][0])\n",
    "    fragmentAndSideTrend[fragmanetAndSideKey] = sorted(fragmentAndSideTrend[fragmanetAndSideKey], key=itemgetter(1))\n",
    "    firstPrev = 0\n",
    "    secondPrev = 0\n",
    "    trend = 0\n",
    "    for match in fragmentAndSideTrend[fragmanetAndSideKey]:\n",
    "        if (match[1] - firstPrev) <= 1:\n",
    "            if match[0] and (secondPrev == 0 or (secondPrev - match[2]) <= 1): # match[0] == inverted\n",
    "                trend += 1\n",
    "            elif (match[2] - secondPrev) <= 1:\n",
    "                trend += 1\n",
    "        firstPrev = match[1]\n",
    "        secondPrev = match[2]\n",
    "    fragmentAndSideTrendVote[fragmanetAndSideKey] = trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-21T13:54:45.865740Z",
     "start_time": "2019-10-21T13:54:42.116511Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# CALC strict trend voting PLUS bonus for synchronized trend ALSO calc bonus separately\n",
    "fragmentAndSideTrendVoteStrict = {}\n",
    "fragmentAndSideTrendVoteSync = {}\n",
    "for fragmanetAndSideKey in fragmentAndSideTrend:\n",
    "    # import pdb; pdb.set_trace()\n",
    "    fragmentAndSideTrend[fragmanetAndSideKey] = sorted(fragmentAndSideTrend[fragmanetAndSideKey], key=itemgetter(2), reverse=fragmentAndSideTrend[fragmanetAndSideKey][0][0])\n",
    "    fragmentAndSideTrend[fragmanetAndSideKey] = sorted(fragmentAndSideTrend[fragmanetAndSideKey], key=itemgetter(1))\n",
    "    firstPrev = -1\n",
    "    secondPrev = -1\n",
    "    trend = 0\n",
    "    sync = 0\n",
    "    maxTrend = 0\n",
    "    for match in fragmentAndSideTrend[fragmanetAndSideKey]:\n",
    "        if firstPrev != -1:\n",
    "            if (match[1] - firstPrev) == 0:\n",
    "                if match[0] and (secondPrev - match[2]) == 1: # match[0] == inverted\n",
    "                    trend += 1\n",
    "                elif (match[2] - secondPrev) == 1:\n",
    "                    trend += 1\n",
    "                else:\n",
    "                    trend = 0\n",
    "            elif (match[1] - firstPrev) == 1:\n",
    "                if match[0] and (secondPrev - match[2]) == 1: # match[0] == inverted\n",
    "                    trend += 2\n",
    "                    sync += 1\n",
    "                elif match[0] and (secondPrev - match[2]) == 0: # match[0] == inverted\n",
    "                    trend += 1\n",
    "                elif (match[2] - secondPrev) == 1:\n",
    "                    trend += 2\n",
    "                    sync += 1\n",
    "                elif (match[2] - secondPrev) == 0:\n",
    "                    trend += 1\n",
    "                else:\n",
    "                    trend = 0\n",
    "        maxTrend = max(trend, maxTrend)\n",
    "        firstPrev = match[1]\n",
    "        secondPrev = match[2]\n",
    "    fragmentAndSideTrendVoteStrict[fragmanetAndSideKey] = maxTrend\n",
    "    fragmentAndSideTrendVoteSync[fragmanetAndSideKey] = sync"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-21T13:55:01.698109Z",
     "start_time": "2019-10-21T13:54:45.867724Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# WRITE results to a CSV file\n",
    "with open('20191021_FRONT_TRAINED_FRONT_pairs_votes.csv', 'w') as csvfile:\n",
    "    csvwriter = csv.writer(csvfile, delimiter=',')\n",
    "    csvwriter.writerow([\"fragmentAndSide\", \n",
    "                        \"fragment\", \n",
    "                        \"fragmentTotal\",\n",
    "                        \"fragmentVote\",\n",
    "                        \"devideVoteByTotal\",\n",
    "                        \"fragmentAndSideTotal\",\n",
    "                        \"fragmentAndSideVote\",\n",
    "                        \"devideSideVoteBySideTotal\",\n",
    "                        \"fragmentAndSideTrendVote\",\n",
    "                        \"devideSideTrendVoteBySideTotal\",\n",
    "                        \"fragmentAndSideTrendVoteStrict\",\n",
    "                        \"devideSideTrendVoteStrictBySideTotal\",\n",
    "                        \"fragmentAndSideTrendVoteSync\",\n",
    "                        \"devideSideTrendVoteSyncBySideTotal\",\n",
    "\n",
    "                        \"firstFileName\",\n",
    "                        \"firstCroppedWidth\",\n",
    "                        \"firstOffsetX\",\n",
    "                        \"firstOffsetY\",\n",
    "                        \"firstHorizontalFlip\",\n",
    "                        \"secondFileName\",\n",
    "                        \"secondCroppedWidth\",\n",
    "                        \"secondOffsetX\",\n",
    "                        \"secondOffsetY\",\n",
    "                        \"secondHorizontalFlip\",                        \n",
    "                        \n",
    "                        \"fragmentAndSideTrend\",\n",
    "                        \"fragmentAndSideCubes\",\n",
    "                        \"fragmentAndSideDrawRect\",\n",
    "                        \"fragmentAndSideMatchPoint\",\n",
    "                        \"origCoordinates\",\n",
    "                        \"class\"\n",
    "                       ])\n",
    "    for fragmanetAndSideKey in fragmentAndSideVote:\n",
    "        csvwriter.writerow([fragmanetAndSideKey, \n",
    "                            fragmentNames[fragmanetAndSideKey], \n",
    "                            fragmentTotal[fragmentNames[fragmanetAndSideKey]],\n",
    "                            fragmentVote[fragmentNames[fragmanetAndSideKey]],\n",
    "                            fragmentVote[fragmentNames[fragmanetAndSideKey]] / fragmentTotal[fragmentNames[fragmanetAndSideKey]],\n",
    "                            fragmentAndSideTotal[fragmanetAndSideKey],\n",
    "                            fragmentAndSideVote[fragmanetAndSideKey],\n",
    "                            fragmentAndSideVote[fragmanetAndSideKey] / fragmentAndSideTotal[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVote[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVote[fragmanetAndSideKey] / fragmentAndSideTotal[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVoteStrict[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVoteStrict[fragmanetAndSideKey] / fragmentAndSideTotal[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVoteSync[fragmanetAndSideKey],\n",
    "                            fragmentAndSideTrendVoteSync[fragmanetAndSideKey] / fragmentAndSideTotal[fragmanetAndSideKey],\n",
    "                            \n",
    "                            firstNames[fragmanetAndSideKey],\n",
    "                            0,\n",
    "                            0,\n",
    "                            0,\n",
    "                            0,\n",
    "                            secondNames[fragmanetAndSideKey],\n",
    "                            0,\n",
    "                            0,\n",
    "                            0,\n",
    "                            0,\n",
    "                            \n",
    "                            fragmentAndSideTrend[fragmanetAndSideKey],\n",
    "                            fragmentAndSideCubes[fragmanetAndSideKey],\n",
    "                            fragmentAndSideDrawRect[fragmanetAndSideKey],\n",
    "                            fragmentAndSideMatchPoint[fragmanetAndSideKey],\n",
    "                            origCoordinates[fragmanetAndSideKey],\n",
    "                            fragmentAndSideClass[fragmanetAndSideKey]\n",
    "                           ])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T07:45:55.519978Z",
     "start_time": "2019-04-04T07:45:55.516181Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# STOP HERE? NO!! Need the next 2 cells for the alignment of the flipped image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T16:43:57.749918Z",
     "start_time": "2019-10-30T16:43:57.731329Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# UTIL 2 - needed functions for flipping and matching\n",
    "def find_match(big_img, small_img):\n",
    "    match_map = match_template(big_img, small_img)\n",
    "    offsets_arr = np.unravel_index(np.argmax(match_map), match_map.shape)\n",
    "    offset_x, offset_y = offsets_arr[::-1]        \n",
    "    return offset_x, offset_y, match_map[offsets_arr]\n",
    "  \n",
    "\n",
    "def extract_image_derivatives(storage_client, chunk, file_name):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    image = load_img_for_name_FRONT(storage_client, chunk, file_name)\n",
    "    short_name = file_name\n",
    "    img_crop = crops[short_name]\n",
    "    cropped = crop(image, img_crop['y_start']-cube_size, img_crop['y_end']+cube_size, \\\n",
    "                   img_crop['x_start']-cube_size, img_crop['x_end']+cube_size)\n",
    "    front = load_front_for_name_FRONT(storage_client, chunk, file_name)\n",
    "    front_adaptive = threshold_adaptive(front, 251, offset=5)  \n",
    "    # import pdb; pdb.set_trace()\n",
    "    cropped_adaptive = threshold_adaptive(cropped, 251, offset=5)\n",
    "    cropped_flipped_lr = cropped_adaptive[:,::-1]\n",
    "    offset_x_lr, offset_y_lr, val_lr = find_match(front_adaptive, cropped_flipped_lr)\n",
    "    cropped_flipped_ud = cropped_adaptive[::-1]\n",
    "    offset_x_ud, offset_y_ud, val_ud = find_match(front_adaptive, cropped_flipped_ud)\n",
    "\n",
    "    if (val_lr >= val_ud):\n",
    "        return cropped_flipped_lr.shape[1], offset_x_lr, offset_y_lr, True\n",
    "    else:\n",
    "        return cropped_flipped_ud.shape[1], offset_x_ud, offset_y_ud, False\n",
    "    \n",
    "\n",
    "def extract_image_derivatives_orig(file_name):\n",
    "    image = load_img_for_name_FRONT(file_name)\n",
    "    split = file_name.split(\"-\")\n",
    "    short_name = split[0] + split[1]\n",
    "    img_crop = crops[short_name]\n",
    "    cropped = crop(image, img_crop['y_start']-cube_size, img_crop['y_end']+cube_size, \\\n",
    "                   img_crop['x_start']-cube_size, img_crop['x_end']+cube_size)\n",
    "    front = load_front_for_name_FRONT(file_name)\n",
    "    front_adaptive = threshold_adaptive(front, 251, offset=5)  \n",
    "    # import pdb; pdb.set_trace()\n",
    "    cropped_adaptive = threshold_adaptive(cropped, 251, offset=5)\n",
    "    cropped_flipped_lr = cropped_adaptive[:,::-1]\n",
    "    offset_x_lr, offset_y_lr, val_lr = find_match(front_adaptive, cropped_flipped_lr)\n",
    "    cropped_flipped_ud = cropped_adaptive[:,::-1]\n",
    "    offset_x_ud, offset_y_ud, val_ud = find_match(front_adaptive, cropped_flipped_ud)\n",
    "\n",
    "    if (val_lr >= val_ud):\n",
    "        return front, cropped_flipped_lr, offset_x_lr, offset_y_lr\n",
    "    else:\n",
    "        return front, cropped_flipped_ud, offset_x_ud, offset_y_ud\n",
    "    \n",
    "    \n",
    "\n",
    "x1start = 0\n",
    "y1start = 0\n",
    "\n",
    "\n",
    "def get_cube_coordinates(file_name, cropped_width, offset_x, offset_y, cubex, cubey):\n",
    "    if file_name[0:file_name.rfind('-D')] not in no_rotate:\n",
    "        temp = cubex\n",
    "        cubex = cropped_width - cubey - cube_size\n",
    "        cubey = temp\n",
    "\n",
    "    reverse_cubex = x1start + offset_x + cube_size + (cropped_width - cubex - cube_size)\n",
    "    reverse_cubey = y1start + offset_y + cube_size + cubey\n",
    "    \n",
    "    return reverse_cubex, reverse_cubey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T07:46:06.696261Z",
     "start_time": "2019-04-04T07:46:05.505565Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Duplicate cell - Can be deleted?\n",
    "# WRITE results for flip and match into a CSV file - WARNING - SLOW RUN!!! \n",
    "import multiprocessing\n",
    "num_cores = 6 # multiprocessing.cpu_count()\n",
    "num_partitions = num_cores * 4\n",
    "\n",
    "def parallelize_dataframe(df, func):\n",
    "    df_split = np.array_split(df, num_partitions)\n",
    "\n",
    "    # test block for running serialy    \n",
    "    # results = []\n",
    "    # for item in zip(np.arange(num_partitions), df_split):\n",
    "    #    results.append(func(item))\n",
    "    # df = pd.concat(results, ignore_index=True)\n",
    "\n",
    "    # parallel block\n",
    "    pool = multiprocessing.Pool(num_cores)\n",
    "    df = pd.concat(pool.map(func, zip(np.arange(num_partitions), df_split)))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return df\n",
    "    \n",
    "def flip_row(storage_client, chunk, df, idx, df_row):\n",
    "    first_file_name = df_row[\"firstFileName\"]\n",
    "    second_file_name = df_row[\"secondFileName\"]\n",
    "    print(\"Process chunk:\" + str(chunk) + \" IDX:\" + str(idx) + \" File1:\" + first_file_name + \" File2:\", second_file_name)\n",
    "\n",
    "    first_cropped_width, first_offset_x, first_offset_y, first_is_horiz = \\\n",
    "        extract_image_derivatives(storage_client, chunk, first_file_name)\n",
    "    second_cropped_width, second_offset_x, second_offset_y, second_is_horiz = \\\n",
    "        extract_image_derivatives(storage_client, chunk, second_file_name)\n",
    "\n",
    "    df.at[idx, \"firstCroppedWidth\"] = first_cropped_width\n",
    "    df.at[idx, \"firstOffsetX\"] = first_offset_x\n",
    "    df.at[idx, \"firstOffsetY\"] = first_offset_y\n",
    "    df.at[idx, \"firstHorizontalFlip\"] = 1 if first_is_horiz else 0\n",
    "\n",
    "    df.at[idx, \"secondCroppedWidth\"] = second_cropped_width\n",
    "    df.at[idx, \"secondOffsetX\"] = second_offset_x\n",
    "    df.at[idx, \"secondOffsetY\"] = second_offset_y\n",
    "    df.at[idx, \"secondHorizontalFlip\"] = 1 if second_is_horiz else 0\n",
    "\n",
    "def flip_df(args):\n",
    "    iter_count = args[0]\n",
    "    split_df = args[1]\n",
    "    print(\"Chunk\", iter_count, \" LEN SPLIT:\", len(split_df.index))\n",
    "    # print(\"Chunk\", iter_count)\n",
    "    storage_client = storage.Client()\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for idx, row in split_df.iterrows():\n",
    "        flip_row(storage_client, iter_count, split_df, idx, row)\n",
    "    return split_df\n",
    "\n",
    "all_matches = pd.read_csv('20190407_pairs_votes.csv')\n",
    "print(\"LEN:\",len(all_matches.index))\n",
    "flipped_matches = parallelize_dataframe(all_matches, flip_df)\n",
    "# import pdb; pdb.set_trace()\n",
    "print(\"LEN FLIP:\",len(flipped_matches.index))\n",
    "flipped_matches.to_csv('20190407_pairs_flipped.csv', index=False)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-10-30T16:49:34.007Z"
    },
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LEN: 69\n",
      ">>>>>>>>>>>starting at: 0\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Process chunk:3 IDX:3 File1:P597Fg028 File2: P598Fg028\n",
      "Process chunk:1 IDX:1 File1:P597Fg037 File2: P597Fg045\n",
      "Process chunk:2 IDX:2 File1:P596Fg001 File2: P597Fg045\n",
      "Process chunk:0 IDX:0 File1:P597Fg069 File2: P598Fg028\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 4\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Process chunk:2 IDX:6 File1:P596Fg001 File2: P598Fg028\n",
      "Process chunk:3 IDX:7 File1:P596Fg001 File2: P597Fg045\n",
      "Process chunk:1 IDX:5 File1:P597Fg028 File2: P597Fg045\n",
      "Process chunk:0 IDX:4 File1:P597Fg069 File2: P598Fg018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 8\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Process chunk:0 IDX:8 File1:P597Fg013 File2: P597Fg045\n",
      "Process chunk:3 IDX:11 File1:P597Fg038 File2: P597Fg045\n",
      "Process chunk:1 IDX:9 File1:P596Fg001 File2: P597Fg071\n",
      "Process chunk:2 IDX:10 File1:P597Fg028 File2: P597Fg045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 12\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Process chunk:0 IDX:12 File1:P597Fg069 File2: P597Fg071\n",
      "Process chunk:1 IDX:13 File1:P597Fg013 File2: P597Fg045\n",
      "Process chunk:2 IDX:14 File1:P597Fg028 File2: P597Fg067\n",
      "Process chunk:3 IDX:15 File1:P590Fg008 File2: P597Fg045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 16\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Process chunk:1 IDX:17 File1:P596Fg001 File2: P597Fg006\n",
      "Process chunk:0 IDX:16 File1:P590Fg008 File2: P590Fg009\n",
      "Process chunk:3 IDX:19 File1:P597Fg069 File2: P598Fg015\n",
      "Process chunk:2 IDX:18 File1:P596Fg001 File2: P597Fg067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 20\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Process chunk:1 IDX:21 File1:P597Fg038 File2: P597Fg045\n",
      "Process chunk:3 IDX:23 File1:P597Fg018 File2: P597Fg045\n",
      "Process chunk:2 IDX:22 File1:P597Fg028 File2: P597Fg044\n",
      "Process chunk:0 IDX:20 File1:P597Fg028 File2: P597Fg062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 24\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Process chunk:3 IDX:27 File1:P597Fg028 File2: P597Fg071\n",
      "Process chunk:2 IDX:26 File1:P597Fg023 File2: P597Fg045\n",
      "Process chunk:0 IDX:24 File1:P597Fg069 File2: P598Fg007\n",
      "Process chunk:1 IDX:25 File1:P597Fg028 File2: P597Fg060\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 28\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Process chunk:3 IDX:31 File1:P597Fg028 File2: P597Fg029\n",
      "Process chunk:2 IDX:30 File1:P597Fg038 File2: P597Fg046\n",
      "Process chunk:0 IDX:28 File1:P597Fg018 File2: P598Fg028\n",
      "Process chunk:1 IDX:29 File1:P597Fg018 File2: P597Fg045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 32\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Process chunk:3 IDX:35 File1:P590Fg008 File2: P597Fg046\n",
      "Process chunk:0 IDX:32 File1:P597Fg033 File2: P597Fg045\n",
      "Process chunk:2 IDX:34 File1:P597Fg018 File2: P597Fg062\n",
      "Process chunk:1 IDX:33 File1:P597Fg062 File2: P597Fg067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>LEN FLIP: 4\n",
      ">>>>>>>>>>>starting at: 36\n",
      "Chunk 3  LEN SPLIT: 1\n",
      "Chunk 2  LEN SPLIT: 1\n",
      "Chunk 0  LEN SPLIT: 1\n",
      "Chunk 1  LEN SPLIT: 1\n",
      "Process chunk:3 IDX:39 File1:P597Fg018 File2: P597Fg071\n",
      "Process chunk:0 IDX:36 File1:P597Fg028 File2: P598Fg036\n",
      "Process chunk:2 IDX:38 File1:P597Fg038 File2: P597Fg067\n",
      "Process chunk:1 IDX:37 File1:P597Fg037 File2: P598Fg015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n",
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/filters/thresholding.py:229: skimage_deprecation: Function ``threshold_adaptive`` is deprecated and will be removed in version 0.15. Use ``threshold_local`` instead.\n",
      "  def threshold_adaptive(image, block_size, method='gaussian', offset=0,\n"
     ]
    }
   ],
   "source": [
    "# WRITE results for flip and match into a CSV file - WARNING - SLOW RUN!!! \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import multiprocessing\n",
    "num_cores = 4 # multiprocessing.cpu_count()\n",
    "num_partitions = num_cores * 40\n",
    "\n",
    "def parallelize_dataframe(df, func, start):\n",
    "    df_split = np.array_split(df, num_partitions)\n",
    "\n",
    "    # test block for running serialy - comment the next block if running this   \n",
    "    # results = []\n",
    "    # for item in zip(np.arange(num_partitions), df_split):\n",
    "    #      results.append(func(item))\n",
    "    # df = pd.concat(results, ignore_index=True)\n",
    "\n",
    "    # import pdb; pdb.set_trace()\n",
    "\n",
    "    # parallel block - comment the previous block if running this\n",
    "    pool = multiprocessing.Pool(num_cores) # num_cores - should be...\n",
    "    df = pd.concat(pool.map(func, zip(np.arange(num_partitions), df_split[start:start+num_cores])))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return df\n",
    "    \n",
    "def flip_row(storage_client, chunk, df, idx, df_row):\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    first_file_name = df_row[\"firstFileName\"]\n",
    "    second_file_name = df_row[\"secondFileName\"]\n",
    "    print(\"Process chunk:\" + str(chunk) + \" IDX:\" + str(idx) + \" File1:\" + first_file_name + \" File2:\", second_file_name)\n",
    "\n",
    "    first_cropped_width, first_offset_x, first_offset_y, first_is_horiz = \\\n",
    "        extract_image_derivatives(storage_client, chunk, first_file_name)\n",
    "    second_cropped_width, second_offset_x, second_offset_y, second_is_horiz = \\\n",
    "        extract_image_derivatives(storage_client, chunk, second_file_name)\n",
    "\n",
    "    df.at[idx, \"firstCroppedWidth\"] = first_cropped_width\n",
    "    df.at[idx, \"firstOffsetX\"] = first_offset_x\n",
    "    df.at[idx, \"firstOffsetY\"] = first_offset_y\n",
    "    df.at[idx, \"firstHorizontalFlip\"] = 1 if first_is_horiz else 0\n",
    "\n",
    "    df.at[idx, \"secondCroppedWidth\"] = second_cropped_width\n",
    "    df.at[idx, \"secondOffsetX\"] = second_offset_x\n",
    "    df.at[idx, \"secondOffsetY\"] = second_offset_y\n",
    "    df.at[idx, \"secondHorizontalFlip\"] = 1 if second_is_horiz else 0\n",
    "\n",
    "def flip_df(args):\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    iter_count = args[0]\n",
    "    split_df = args[1]\n",
    "    print(\"Chunk\", iter_count, \" LEN SPLIT:\", len(split_df.index))\n",
    "    # print(\"Chunk\", iter_count)\n",
    "    storage_client = storage.Client()\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for idx, row in split_df.iterrows():\n",
    "        flip_row(storage_client, iter_count, split_df, idx, row)\n",
    "    return split_df\n",
    "\n",
    "all_matches = pd.read_csv('FRONT_back_trained_front_matched_final.csv')\n",
    "\n",
    "# RUN SINGLE_THREAD - for DEBUG only - Better utilize the commented block inside the parallelize method!!!\n",
    "# for idx, row in all_matches.iterrows():\n",
    "#     import pdb; pdb.set_trace()\n",
    "#     flip_row(storage_client, 1, all_matches, idx, row)\n",
    "\n",
    "# RUN MULTI_THREAD    \n",
    "print(\"LEN:\",len(all_matches.index))\n",
    "length = len(all_matches.index)\n",
    "for start in range(0, length, num_cores):    \n",
    "    print(\">>>>>>>>>>>starting at: \"+str(start))\n",
    "    flipped_matches = parallelize_dataframe(all_matches, flip_df, start)\n",
    "    #import pdb; pdb.set_trace()\n",
    "    print(\">>>>>>>>>>>LEN FLIP:\",len(flipped_matches))\n",
    "    flipped_matches.to_csv('FRONT_back_trained_front_matched_final_flipped_'+str(start)+'.csv')   \n",
    "\n",
    "warnings.filterwarnings(\"default\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# STOP!!! Obselete!!! WRITE results to a CSV file\n",
    "with open('cubes_X3_e.csv', 'w') as csvfile:\n",
    "    csvwriter = csv.writer(csvfile, delimiter=',')\n",
    "    csvwriter.writerow([\"fragmanetAndSide\", \n",
    "                        \"fragment\", \n",
    "                        \"fragmentVote\",\n",
    "                        \"fragmentAndSideVote\",\n",
    "                        \"fragmentAndSideTrendVote\",\n",
    "                        \"fragmentAndSideTrendVoteStrict\",\n",
    "                        \"fragmentAndSideTrendVoteSync\",\n",
    "                        \"firstFileName\",\n",
    "                        \"firstCroppedWidth\",\n",
    "                        \"firstOffsetX\",\n",
    "                        \"firstOffsetY\",\n",
    "                        \"firstHorizontalFlip\",\n",
    "                        \"secondFileName\",\n",
    "                        \"secondCroppedWidth\",\n",
    "                        \"secondOffsetX\",\n",
    "                        \"secondOffsetY\",\n",
    "                        \"secondHorizontalFlip\",\n",
    "                        \"fragmentAndSideTrend\",\n",
    "                        \"fragmentAndSideCubes\",\n",
    "                        \"origCoordinates\"])\n",
    "    for fragmanetAndSideKey in fragmentAndSideVote:\n",
    "        if fragmentAndSideTrendVoteSync[fragmanetAndSideKey] > 0:\n",
    "            print(fragmanetAndSideKey)\n",
    "\n",
    "            first_file_name = matchFirstFile[fragmanetAndSideKey]\n",
    "            first_cropped_width, first_offset_x, first_offset_y, first_is_horiz = \\\n",
    "                extract_image_derivatives(first_file_name)\n",
    "            second_file_name = matchSecondFile[fragmanetAndSideKey]\n",
    "            second_cropped_width, second_offset_x, second_offset_y, second_is_horiz = \\\n",
    "                extract_image_derivatives(second_file_name)\n",
    "\n",
    "            csvwriter.writerow([fragmanetAndSideKey, \n",
    "                                fragmentNames[fragmanetAndSideKey], \n",
    "                                fragmentVote[fragmentNames[fragmanetAndSideKey]],\n",
    "                                fragmentAndSideVote[fragmanetAndSideKey],\n",
    "                                fragmentAndSideTrendVote[fragmanetAndSideKey],\n",
    "                                fragmentAndSideTrendVoteStrict[fragmanetAndSideKey],\n",
    "                                fragmentAndSideTrendVoteSync[fragmanetAndSideKey],\n",
    "                                first_file_name,\n",
    "                                first_cropped_width,\n",
    "                                first_offset_x, \n",
    "                                first_offset_y,\n",
    "                                first_is_horiz,\n",
    "                                second_file_name,\n",
    "                                second_cropped_width,\n",
    "                                second_offset_x, \n",
    "                                second_offset_y, \n",
    "                                second_is_horiz,\n",
    "                                fragmentAndSideTrend[fragmanetAndSideKey],\n",
    "                                fragmentAndSideCubes[fragmanetAndSideKey],\n",
    "                                origCoordinates[fragmanetAndSideKey]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# DEBUG - don't run\n",
    "\n",
    "done = False\n",
    "for fragmanetAndSideKey in fragmentAndSideVote:\n",
    "    if fragmentAndSideTrendVoteSync[fragmanetAndSideKey] > 0 and not done:\n",
    "        # import pdb; pdb.set_trace()\n",
    "        file_name = matchFirstFile[fragmanetAndSideKey]\n",
    "        print(fragmanetAndSideKey)\n",
    "        print(file_name)\n",
    "        cropped_width, offset_x, offset_y = extract_image_derivatives(file_name)\n",
    "        done = True\n",
    "        front = load_front_for_name(file_name)\n",
    "        for cube_match in fragmentAndSideCubes[fragmanetAndSideKey]:\n",
    "            x_co, y_co = get_cube_coordinates(file_name, cropped_width, offset_x, offset_y, cube_match[0], cube_match[1])\n",
    "            fig = plt.figure(figsize=(20, 6))\n",
    "            ax2 = plt.subplot(1, 3, 1, adjustable='box-forced')\n",
    "            ax2.imshow(front, cmap=plt.cm.gray)\n",
    "            rect = plt.Rectangle((x_co, y_co), cube_size, cube_size, edgecolor='r', facecolor='none')\n",
    "            ax2.add_patch(rect)\n",
    "            plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# DEBUG - don't run\n",
    "for cube_match in fragmentAndSideCubes[fragmanetAndSideKey]:\n",
    "    draw_on_plot(plt, ax2, flipped, offset_x, offset_y, cube_match[0], cube_match[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-13T11:29:43.847201Z",
     "start_time": "2019-04-13T11:29:43.343685Z"
    }
   },
   "outputs": [],
   "source": [
    "img_path = '/media/1KGB_ILAN/papyrus/files/temp/'\n",
    "img_path = 'thesis-papyri/PAPYRI/P589/P589-Fg001-V/'\n",
    "for file_ in list_blobs_with_prefix(storage_client, img_path):\n",
    "    #print(file_)\n",
    "    if (\" _018\" in file_):        \n",
    "        print(file_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-27T11:47:22.185514Z",
     "start_time": "2019-09-27T11:47:22.177151Z"
    }
   },
   "outputs": [],
   "source": [
    "Out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}