17_Process_training_tears_using_cubes_match.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.432015Z",
     "start_time": "2019-10-30T20:58:13.315525Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Main import block and TODO list\n",
    "\n",
    "# TODO: see how uri calculated the ridges\n",
    "\n",
    "# TODO: Perform Histogram equalization - start with it\n",
    "# TODO: \n",
    "# take integral from the Highest peak+-0.005 divide by integral of the entire graph \n",
    "# This will be the peakness measure for the PSD ==> The desired ridge index\n",
    "# TODO:\n",
    "# take integral from the Highest peak+-0.005 divide by integral of the entire graph - it's the peakness measure for the PSD\n",
    "# must select a peak above a min threshold in order to ignore noisy frequency\n",
    "# must ignore peaks above a certain threshold in order to detect meaningful frequency\n",
    "# run the PSD in moving windows every 200 px (deduced from the below PSD pointing to a freq of 1/0.02=50-> times 4= 200px)\n",
    "# and medianf the result of the windows\n",
    "# TODO:\n",
    "# Another alternative: (with Yariv)\n",
    "# Run PSD column by column - get the phase, freq, peakness and reconstruct an artificial ridge slice\n",
    "# from this - reconstruct a \"clean\" artificial ridge image\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.image as img\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import urllib.request\n",
    "import os\n",
    "import shutil\n",
    "import glob\n",
    "from scipy import ndimage\n",
    "from scipy import signal\n",
    "#import cv2\n",
    "\n",
    "from PIL import Image, ImageDraw, ImageFont\n",
    "\n",
    "import mahotas as mh\n",
    "from mahotas import polygon\n",
    "# import pymorph as pm\n",
    "\n",
    "import networkx as nx\n",
    "\n",
    "from scipy import ndimage as nd\n",
    "import skimage.transform as transform\n",
    "import skimage.morphology as mp\n",
    "import skimage.io as sio\n",
    "import scipy.misc as sm\n",
    "from skimage.filters import threshold_otsu, threshold_adaptive\n",
    "from skimage.feature import hessian_matrix, hessian_matrix_eigvals\n",
    "from skimage import exposure\n",
    "from skimage import data, img_as_float\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "from bisect import bisect_left\n",
    "import math\n",
    "import warnings\n",
    "import csv\n",
    "import tensorflow as tf\n",
    "from time import gmtime, strftime\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.574242Z",
     "start_time": "2019-10-30T20:58:15.434040Z"
    },
    "code_folding": [
     0,
     28,
     31,
     49,
     57,
     71,
     77,
     83,
     98,
     107,
     122,
     140,
     157,
     175,
     276,
     286,
     296,
     306,
     318,
     323,
     328,
     335,
     357,
     364,
     373,
     410
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Utility functions\n",
    "\n",
    "# One time init\n",
    "# with open('results.csv', 'w') as csvfile:\n",
    "#     csvout = csv.writer(csvfile)\n",
    "#     csvout.writerow([\"File\", \"Model\", \"Gap\", \"Slice_size\", \"Count\", \"Precision\", \"Recall\", \"F-score\", \"True Count\", \"Error Rate\"])\n",
    "\n",
    "#BASIC CROP FRAME\n",
    "X_START = 1000\n",
    "X_END = 6000\n",
    "Y_START = 800\n",
    "Y_END = 4300\n",
    "BG_2_OBJ_RATIO = 0.91\n",
    "CUBE_SIZE = 250\n",
    "EDGE_GAP = 50\n",
    "ROOT_FOLDER = \"/media/1KGB_ILAN/papyrus/\"\n",
    "fragments_folder = \"files/fragments_train_validation/\"\n",
    "# ROOT_FOLDER = \"/Users/il239838/Downloads/private/Thesis/Papyrus/PX303/files/\"\n",
    "LEARNING_RATE = 0.001\n",
    "BATCHES = 2000\n",
    "BATCH_SIZE = 60\n",
    "BREAK_VAL = 1000\n",
    "\n",
    "# Simple crop by x/y ranges\n",
    "def crop(image, ymin, ymax, xmin, xmax):\n",
    "    return image[ymin:ymax, xmin:xmax]\n",
    "\n",
    "# returns a logical matrix of values beyond a threshld\n",
    "def thresholded(image, val): \n",
    "    return np.logical_and(*[image[...] > val  for t in enumerate([0, 0])])\n",
    "\n",
    "def find_min_max_without_orphand_pixels(nonzero_dimension, crop_filter=20):\n",
    "    sorted = np.sort(nonzero_dimension)\n",
    "    prev=-1\n",
    "    min_val = sorted[0]\n",
    "    for i, x in enumerate(sorted[:100]):\n",
    "        if prev >= 0 and x - prev > crop_filter:\n",
    "            min_val = x\n",
    "        prev = x\n",
    "    prev=-1\n",
    "    max_val = sorted[-1]\n",
    "    for i, x in enumerate(sorted[-100:]):\n",
    "        if prev >= 0 and x - prev > crop_filter:\n",
    "            max_val = prev\n",
    "            break\n",
    "        prev = x\n",
    "    \n",
    "    return min_val, max_val\n",
    "\n",
    "def calc_min_max_coordinates(image, crop_val=50):\n",
    "    temp = thresholded(image, crop_val)\n",
    "    temp = temp * 1\n",
    "    temp = np.nonzero(temp)\n",
    "    ymin, ymax = find_min_max_without_orphand_pixels(temp[0])\n",
    "    xmin,xmax = find_min_max_without_orphand_pixels(temp[1])\n",
    "    return ymin, ymax, xmin, xmax\n",
    "\n",
    "def calc_min_max_coordinates_dynamic(image, cutoff=1):\n",
    "    temp = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "    flat = np.sort(np.matrix.getA1(temp))\n",
    "    sum_all = np.sum(flat)\n",
    "    index = np.argmin(flat.cumsum() < (sum_all * cutoff))\n",
    "\n",
    "    temp = thresholded(temp, flat[index])\n",
    "    temp = temp * 1\n",
    "    temp = np.nonzero(temp)\n",
    "    ymin, ymax = find_min_max_without_orphand_pixels(temp[0])\n",
    "    xmin,xmax = find_min_max_without_orphand_pixels(temp[1])\n",
    "    return ymin, ymax, xmin, xmax\n",
    "\n",
    "# initial static crop and a seondary dynamic crop based on signal2noise ratio\n",
    "def crop_full_scan(image, x_start, x_end, y_start, y_end):\n",
    "    temp = crop(image, y_start, y_end, x_start, x_end)\n",
    "    ymin, ymax, xmin, xmax = calc_min_max_coordinates_dynamic(temp, cutoff=BG_2_OBJ_RATIO)\n",
    "    temp = crop(image, y_start+ymin, y_start+ymax, x_start+xmin, x_start+xmax)\n",
    "    return temp\n",
    "\n",
    "def crop_thresholded(image):\n",
    "    temp = crop(image, 0, image.shape[0]-1, 0, image.shape[1]-1)\n",
    "    ymin, ymax, xmin, xmax = calc_min_max_coordinates(temp)\n",
    "    temp = crop(image, ymin, ymax, xmin, xmax)\n",
    "    return temp\n",
    "\n",
    "def read_and_crop(image_name, x_start=X_START, x_end=X_END, y_start=Y_START, y_end=Y_END):\n",
    "    #if \"il239838\" in os.getcwd():\n",
    "    #    image = img.imread(ROOT_FOLDER + image_name)\n",
    "    #else:\n",
    "    #    f = urllib.request.urlopen(\"https://dl.dropboxusercontent.com/s/31b96942qdcn73k/\" + image_name)\n",
    "    #    image = img.imread(f, format='jpeg')\n",
    "    image = img.imread(ROOT_FOLDER + image_name)\n",
    "    \n",
    "    # Smart-crop the image to get rid of all the noise and redundant area\n",
    "    # return crop_full_scan(image)\n",
    "    cropped = crop_full_scan(image, x_start, x_end, y_start, y_end)\n",
    "    return exposure.equalize_adapthist(cropped, clip_limit=0.03)\n",
    "\n",
    "\n",
    "# TODO: fix performance!!! http://scikit-image.org/docs/dev/user_guide/tutorial_parallelization.html\n",
    "def combine_3_images_to_RGB(red, green, blue):\n",
    "    new_image = np.empty((blue.shape[0],blue.shape[1],3))\n",
    "    for x in range(0, blue.shape[0]):\n",
    "        for y in range(0, blue.shape[1]):\n",
    "            new_image[x,y,0] = red[x,y]\n",
    "            new_image[x,y,1] = green[x,y]\n",
    "            new_image[x,y,2] = blue[x,y]\n",
    "    return new_image\n",
    "\n",
    "def slice_image_left_edge(original, width=200, rotate=0):\n",
    "    rot = ndimage.rotate(original, rotate)\n",
    "    # Slice the left slice of the so-called \"blue\" image\n",
    "    left_edge_orig = crop(rot, 1, 1400, 1, width)\n",
    "    left_edge_orig = crop_thresholded(left_edge_orig)\n",
    "\n",
    "    # Copy to a new array so we don't thrash the origin\n",
    "    left_edge = np.empty_like (left_edge_orig)\n",
    "    np.copyto(left_edge, left_edge_orig)\n",
    "\n",
    "    # Zero down low level \"noise\" values\n",
    "    low_values_indices = left_edge < 30  # Where values are low\n",
    "    left_edge[low_values_indices] = 0  # All low values set to 0\n",
    "    return left_edge\n",
    "\n",
    "def get_best_angle_rotation(original, crop=True, width=200):\n",
    "    min_var = 99999999999\n",
    "    best_angle = -10\n",
    "    for x in range(-5,5):\n",
    "        if crop:            \n",
    "            rot_edge = slice_image_left_edge(original, width, x)\n",
    "        else:\n",
    "            rot_edge = ndimage.rotate(original, x)\n",
    "        left_var = np.var(rot_edge, axis=1)\n",
    "        # left_var = np.apply_along_axis(lambda v: np.var(v[np.nonzero(v)]), 1, rot_edge)\n",
    "        var_sum = np.sum(left_var)\n",
    "        if (var_sum < min_var):\n",
    "            min_var = var_sum\n",
    "            best_angle = x\n",
    "    print (\"best_angle=\"+str(best_angle))\n",
    "    return best_angle\n",
    "\n",
    "#     import pdb; pdb.set_trace()\n",
    "def calc_neighbors(slice_map, col, row):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    if ((col-1, row) in slice_map and slice_map[(col-1, row)] != None):\n",
    "        slice_map[(col, row)][\"left\"] = slice_map[(col-1, row)]\n",
    "        slice_map[(col-1, row)][\"right\"] = slice_map[(col, row)]\n",
    "    if ((col+1, row) in slice_map and slice_map[(col+1, row)] != None):\n",
    "        slice_map[(col, row)][\"right\"] = slice_map[(col+1, row)]\n",
    "        slice_map[(col+1, row)][\"left\"] = slice_map[(col, row)]\n",
    "    if ((col, row-1) in slice_map and slice_map[(col, row-1)] != None):\n",
    "        slice_map[(col, row)][\"top\"] = slice_map[(col, row-1)]\n",
    "        slice_map[(col, row-1)][\"bottom\"] = slice_map[(col, row)]\n",
    "    if ((col, row+1) in slice_map and slice_map[(col, row+1)] != None):\n",
    "        slice_map[(col, row)][\"bottom\"] = slice_map[(col, row+1)]\n",
    "        slice_map[(col, row+1)][\"top\"] = slice_map[(col, row)]\n",
    "    \n",
    "\n",
    "\n",
    "def VAL_create_cube(name, raw, x, y):\n",
    "    cube = {}\n",
    "    cube[\"cube\"] = raw\n",
    "    cube[\"file\"] = name\n",
    "    #     if name.find('P') == 0:\n",
    "    #         cube[\"index\"] = int(name[name.find('P')+1:name.find('P')+4]) * 1000 + int(name[name.find('Fg')+2:name.find('Fg')+5])\n",
    "    #     else:\n",
    "    # print(\"Found a ZERO index cube with the name:\"+name)\n",
    "    cube[\"index\"] = 0\n",
    "    cube[\"top_row\"] = x\n",
    "    cube[\"left_col\"] = y\n",
    "    cube[\"right_col\"] = y + CUBE_SIZE\n",
    "    return cube\n",
    "    \n",
    "\n",
    "ZERO_CUBE = VAL_create_cube(\"ZERO\", np.zeros((CUBE_SIZE, CUBE_SIZE), dtype=np.int), -1, -2)\n",
    "\n",
    "# slice an image to cubes with 250X250 pixel size\n",
    "def VAL_slice_TEAR_to_static_slices(name, cropped_original):\n",
    "    structure = {}\n",
    "    # cropped_original = cropped_original / 256 # divide by 256 to \"normalize\" between 0 and 1\n",
    "\n",
    "    x, y = cropped_original[\"cut\"].shape\n",
    "    # print (x,y)\n",
    "    n = 0\n",
    "    # see n offset to see offset in pixels on the x axis == rows. every n equals CUBE_SIZE\n",
    "    while ((n + 1) * CUBE_SIZE < x):\n",
    "        # mark the piece as narrow so the first would be counted as lastt too\n",
    "        narrow = True if ((CUBE_SIZE + (4 * EDGE_GAP)) > y) else False\n",
    "        # cut a cube of 250X250 at the FIRST column\n",
    "        start_row_px = int(np.round(n * CUBE_SIZE, -1))\n",
    "        end_row_px = int(np.round((n + 1) * CUBE_SIZE, -1))\n",
    "\n",
    "        # crop a cube on the starting edge of the current row. \n",
    "        # start with EDGE_GAP and move inwards untill we have value\n",
    "        fg_value = 0\n",
    "        iteration = 1\n",
    "        while fg_value <= 0.2 and iteration <= 5:\n",
    "            cube = (crop(cropped_original[\"cut\"], start_row_px, end_row_px, \\\n",
    "                         EDGE_GAP * iteration, CUBE_SIZE + (EDGE_GAP * iteration)))\n",
    "            iteration += 1\n",
    "            fg_value = np.median(cube)\n",
    "            \n",
    "        # keep only cubes for which half of the pixels have some \"color\"\n",
    "        if np.median(cube) > 0.2: # aligned with the normalization 0.2 correlates to 50\n",
    "            # keep the cube\n",
    "            new_cube = VAL_create_cube(name, cube, start_row_px, EDGE_GAP)\n",
    "            new_cube[\"col\"] = 0 # marks that the cube is on the first col of the piece\n",
    "            new_cube[\"row\"] = n\n",
    "            new_cube[\"last\"] = narrow # marks that the cube is on the last col of the piece\n",
    "            new_cube[\"orig\"] = cropped_original\n",
    "            new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + EDGE_GAP\n",
    "            new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + CUBE_SIZE + EDGE_GAP\n",
    "            new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + start_row_px\n",
    "            new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + end_row_px\n",
    "            structure[(0, n)] = new_cube\n",
    "\n",
    "        # crop a cube on the starting edge of the current row. \n",
    "        # start with EDGE_GAP and move inwards untill we have value\n",
    "        fg_value = 0\n",
    "        iteration = 1\n",
    "        while fg_value <= 0.2 and iteration <= 5:\n",
    "            cube = (crop(cropped_original[\"cut\"], start_row_px, end_row_px, \\\n",
    "                         y - CUBE_SIZE - (EDGE_GAP * iteration), y - (EDGE_GAP * iteration)))\n",
    "            iteration += 1\n",
    "            fg_value = np.median(cube)\n",
    "            \n",
    "        # keep only cubes for which half of the pixels have some \"color\"\n",
    "        # aligned with the normalization 0.2 correlates to 50\n",
    "        if np.median(cube) > 0.2:\n",
    "            # keep the cube\n",
    "            new_cube = VAL_create_cube(name, cube, start_row_px, y - CUBE_SIZE - EDGE_GAP)\n",
    "            new_cube[\"col\"] = 1 # marks that the cube is on the last col of the piece\n",
    "            new_cube[\"row\"] = n\n",
    "            new_cube[\"last\"] = not narrow # like col - marks that the cube is on the last col of the piece\n",
    "            new_cube[\"orig\"] = cropped_original\n",
    "            new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + y - CUBE_SIZE - EDGE_GAP\n",
    "            new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + y - EDGE_GAP\n",
    "            new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + start_row_px\n",
    "            new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + end_row_px\n",
    "            structure[(1, n)] = new_cube\n",
    "\n",
    "    #         m = 0\n",
    "    #         # every 250 pixels on the y axis == cols\n",
    "    #         while ((m + 1) * CUBE_SIZE < y):            \n",
    "    #             if ((m == 0) or ((m + 2) * CUBE_SIZE >= y)): # Only keep the left and right edges of the piece for matching!!\n",
    "    #                 # cut a cube of 250X250\n",
    "    #                 cube = crop(cropped_original[\"cut\"], n * CUBE_SIZE, (n + 1) * CUBE_SIZE, m * CUBE_SIZE, (m + 1) * CUBE_SIZE)\n",
    "    #                 # keep only cubes for which half of the pixels have some \"color\"\n",
    "    #                 # print(np.median(cube))\n",
    "    #                 if np.median(cube) > 0.2: # aligned with the normalization 0.2 correlates to 50\n",
    "    #                     # keep the cube\n",
    "    #                     new_cube = VAL_create_cube(name, cube, n * CUBE_SIZE, m * CUBE_SIZE)\n",
    "    #                     new_cube[\"col\"] = m\n",
    "    #                     new_cube[\"row\"] = n\n",
    "    #                     new_cube[\"orig\"] = cropped_original\n",
    "    #                     new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + m * CUBE_SIZE\n",
    "    #                     new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + (m + 1) * CUBE_SIZE\n",
    "    #                     new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + n * CUBE_SIZE\n",
    "    #                     new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + (n + 1) * CUBE_SIZE\n",
    "    #                     if ((m + 2) * CUBE_SIZE >= y):\n",
    "    #                         new_cube[\"last\"] = True\n",
    "    #                     else:\n",
    "    #                         new_cube[\"last\"] = False\n",
    "    #                     structure[(m, n)] = new_cube\n",
    "    #             m += 1\n",
    "        n += 0.2 # currently set to jump in 50 px offset\n",
    "        n = np.round(n, 1)\n",
    "        \n",
    "    # this loop has to be performed only after we've established all the None cubes\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for cube in structure.values():\n",
    "        # set the reference to neighbor cubes\n",
    "        if cube != None:\n",
    "            calc_neighbors(structure, cube[\"col\"], cube[\"row\"])\n",
    "            \n",
    "    # return the data structure with all the cubes and the counters of the rows and columns\n",
    "    return structure.values()\n",
    "\n",
    "def pad_above(original, above, amount):\n",
    "    res = np.insert(original[\"cube\"], np.zeros(amount), above[\"cube\"][-amount:], axis=0)\n",
    "    res = np.delete(res, np.arange(CUBE_SIZE,CUBE_SIZE+amount), axis=0)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"] - amount, original[\"left_col\"])\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"]\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"]\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"] - amount\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"] - amount\n",
    "    return cube\n",
    "\n",
    "def pad_below(original, below, amount):\n",
    "    res = np.insert(original[\"cube\"], np.full(amount, CUBE_SIZE), below[\"cube\"][:amount], axis=0)\n",
    "    res = np.delete(res, np.arange(0, amount), axis=0)\n",
    "    cube =  VAL_create_cube(original[\"file\"], res, original[\"top_row\"] + amount, original[\"left_col\"])\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"]\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"]\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"] + amount\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"] + amount\n",
    "    return cube\n",
    "    \n",
    "def pad_left(original, left, amount):\n",
    "    res = np.insert(original[\"cube\"], np.zeros(amount, dtype=int), left[\"cube\"][:,-amount:], axis=1)\n",
    "    res = np.delete(res, np.arange(CUBE_SIZE, CUBE_SIZE+amount), axis=1)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"], original[\"left_col\"] - amount)\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"] - amount\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"] - amount\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"]\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"]\n",
    "    return cube\n",
    "\n",
    "def pad_right(original, right, amount):\n",
    "    res = np.insert(original[\"cube\"], [CUBE_SIZE], right[\"cube\"][:,:amount], axis=1)\n",
    "    res = np.delete(res, np.arange(0, amount), axis=1)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"], original[\"left_col\"] + amount)\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"] + amount\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"] + amount\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"]\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"]\n",
    "    return cube\n",
    "    \n",
    "\n",
    "# \"Shave\" the right edge of the cube with <gap> pixels and pad with zeros on the left\n",
    "def shave_right(original, amount):\n",
    "    return pad_left(original, ZERO_CUBE, amount)\n",
    "    \n",
    "\n",
    "# \"Shave\" the left edge of the cube with <gap> pixels and pad with zeros on the right    \n",
    "def shave_left(original, amount):\n",
    "    return pad_right(original, ZERO_CUBE, amount)\n",
    "    \n",
    "\n",
    "# concatenate cubes \n",
    "def concatenate_cubes(left, right, slice_size):\n",
    "    con = np.concatenate((left[\"cube\"][:,-slice_size:], right[\"cube\"][:,:slice_size]), axis=1)\n",
    "    x_delta = right[\"top_row\"] - left[\"top_row\"]\n",
    "    y_delta = right[\"left_col\"] - left[\"right_col\"] \n",
    "    return con, x_delta, y_delta\n",
    "\n",
    "# concatenate cubes \n",
    "def VAL_concatenate_cubes(left, right, slice_size):\n",
    "    right_img = right[\"cube\"]\n",
    "    # next block is not relevant for training ...\n",
    "    #     # if the left cube is matched to another left cube (or right cube to another right cube) then rotate the right\n",
    "    #     # cube by 180 so we try to match it upside down, covering the option that the cube was pictured rotated\n",
    "    #     if ((left[\"col\"] == 0 and right[\"col\"] == 0) or (left[\"col\"] != 0 and right[\"col\"] != 0)):\n",
    "    #         right_img = np.rot90(right[\"cube\"], 2);\n",
    "\n",
    "    con = np.concatenate((left[\"cube\"][:,-slice_size:], right_img[:,:slice_size]), axis=1)\n",
    "\n",
    "    # next block calculates distance based on the distance between left's right-top corner and right's left-top corner    \n",
    "    #     x_delta = right[\"top_row\"] - left[\"top_row\"]\n",
    "    #     y_delta = right[\"left_col\"] - left[\"right_col\"] \n",
    "\n",
    "    # next block calculates the distance between the centers of cubes, accounting for test set's possibility of reverse slices (left instead of right and vice versa)\n",
    "    x_delta = right[\"row_px_top\"] - left[\"row_px_top\"] # equivalent to distance between vertical centers\n",
    "    y_delta = (right[\"col_px_left\"] + (slice_size / 2)) - (left[\"col_px_right\"] - (slice_size / 2)) # measuring the distance between horizontal centers of the slices\n",
    "\n",
    "    return con, x_delta, y_delta, left[\"file\"], right[\"file\"]\n",
    "    \n",
    "\n",
    "# concatenate cubes while artificially creating a gap between them. Pad the other end of the cube with zeros\n",
    "def concatenate_cubes_zero_pad_gaps(left_orig, right_orig, gap):\n",
    "    left = left_orig if gap == 0 else shave_right(left_orig, gap)\n",
    "    right = right_orig if gap == 0 else shave_left(right_orig, gap)\n",
    "    return concatenate_cubes(left, right)    \n",
    "\n",
    "# concatenate cubes while artificially creating a gap between them. Pad the other end of the cobe with the nearby\n",
    "# continuation of the cubes\n",
    "def concatenate_cubes_with_gap(left_orig, right_orig, gap, left_pad, right_pad, slice_size):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    left = left_orig if gap == 0 else pad_left(left_orig, left_pad, gap)\n",
    "    right = right_orig if gap == 0 else pad_right(right_orig, right_pad, gap)\n",
    "    return concatenate_cubes(left, right, slice_size)        \n",
    "\n",
    "# convert the data structure of cubes into a train set of 2 arrays of images and labels\n",
    "# each image is a concatanation of 2 images from the original cubes set, covering all combinations of images\n",
    "# effectively creating Nx(N-1) images\n",
    "def VAL_build_train_set_for_euclidean_distance(cubes, slice_size, folder):\n",
    "    # clean folder before starting\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    #import pdb; pdb.set_trace()\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    train_left_obj = []\n",
    "    train_right_obj = []\n",
    "    # iterate over all cubes   \n",
    "    for curr in cubes:\n",
    "        # iterate over the others (effectively n^2)\n",
    "        for adj in cubes:\n",
    "            if (adj[\"file\"] != curr[\"file\"]): # no need to test against self CURRENTLY checking from directions!!!\n",
    "                #import pdb; pdb.set_trace()\n",
    "                # append the adjacent image to the current image\n",
    "                conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr, adj, slice_size)\n",
    "                output = folder+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                np.save(output, conc)\n",
    "                train_imgs.append(output)\n",
    "                train_x_delta.append(x_delta)\n",
    "                train_y_delta.append(y_delta)\n",
    "                train_left_obj.append(curr)\n",
    "                train_right_obj.append(adj)\n",
    "\n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_x_delta, train_y_delta, train_left_obj, train_right_obj\n",
    "\n",
    "\n",
    "# convert the data structure of cubes into a train set of 2 arrays of images and labels\n",
    "# each image is a concatanation of 2 images from the original cubes set, covering all combinations of images\n",
    "# effectively creating Nx(N-1) images\n",
    "def ORIG_build_train_set(cubes, gap):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    # iterate over the rows and cols, essentially going over the grid of sliced cubes\n",
    "    for row in range(0, rows):\n",
    "        for col in range(0, cols):\n",
    "            # if this cube exists (could have been removed previously due to lack of data)\n",
    "            if (cubes[(col, row)] != None):\n",
    "                # for each \"current\" image in the iteration\n",
    "                curr = cubes[(col, row)]\n",
    "                # iterate over all the cubes to find all the \"other\" (adjacent) cubes\n",
    "                for adj_row in range(0, rows):\n",
    "                    for adj_col in range(0, cols):\n",
    "                        if (adj_row != row or adj_col != col):\n",
    "                            if (cubes[(adj_col, adj_row)] != None):\n",
    "                                adj = cubes[(adj_col, adj_row)]\n",
    "                                # append the adjacent image to the current image\n",
    "                                # pass the filling cubes on the right and left to pad against the gap\n",
    "                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                    if (gap == 0):\n",
    "                                        conc, x_delta, y_delta = concatenate_cubes(curr, adj, slice_size)\n",
    "                                    else:\n",
    "                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr, adj, gap, curr[\"left\"], adj[\"right\"], slice_size)\n",
    "                                    train_imgs.append(conc)\n",
    "                                    train_x_delta.append(x_delta)\n",
    "                                    train_y_delta.append(y_delta)\n",
    "                                    # if the adj image is on the same row and on the right of the curr image - it will be marked as match    \n",
    "                                    if (adj_row == row and adj_col == (col + 1)):\n",
    "                                        # mark the image as matched\n",
    "                                        train_lbls.append([0,1])\n",
    "                                        # need to enrich the set with a few more tru positive samples - so we offset \n",
    "                                        # the matched images up ad down a few times and create more matches\n",
    "                                        if (\"top\" in curr.keys() and \"top\"in adj.keys()):\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "                                                adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys() and \"top\" in curr[\"left\"].keys() and \"top\"in curr[\"right\"].keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_above(curr[\"left\"], curr[\"left\"][\"top\"], i) # FIXIT?\n",
    "                                                        adj1Right = pad_above(adj[\"right\"], curr[\"right\"][\"top\"], i) # FIXIT?\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"bottom\" in curr.keys() and \"bottom\"in adj.keys()):\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "                                                adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys() and \"bottom\" in curr[\"left\"].keys() and \"bottom\"in curr[\"right\"].keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_below(curr[\"left\"], curr[\"left\"][\"bottom\"], i) # FIXIT?\n",
    "                                                        adj1Right = pad_below(adj[\"right\"], curr[\"right\"][\"bottom\"], i) # FIXIT?\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"left\" in curr.keys()): # enough to check only the curr as the left of the adj is the curr\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_left(curr, curr[\"left\"],i)\n",
    "                                                adj1 = pad_left(adj, adj[\"left\"],i) # essentially the curr\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_left(curr[\"left\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        adj1Right = pad_left(adj[\"right\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"right\" in adj.keys()): # enough to check only the adj as the right of the curr is the adj\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_right(curr, curr[\"right\"],i) # essentially the adj\n",
    "                                                adj1 = pad_right(adj, adj[\"right\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_right(curr[\"left\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        adj1Right = pad_right(adj[\"right\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                    else:\n",
    "                                        # mark the image as not matched\n",
    "                                        train_lbls.append([1,0])\n",
    "                                \n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.618395Z",
     "start_time": "2019-10-30T20:58:15.575729Z"
    },
    "code_folding": [
     0,
     7,
     248,
     253,
     278,
     297
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# RUN Utility functions 2\n",
    "SAVE_PNG=False\n",
    "def save_img(path, img):\n",
    "    np.save(path, img)  \n",
    "    if SAVE_PNG:\n",
    "        plt.imsave(path+\".png\", img, cmap=plt.cm.gray)\n",
    "                    \n",
    "def VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                              train_y_delta, is_enriched, curr, adj, tolerance_factor=0):\n",
    "    # need to enhance the set with a few more true positive samples\n",
    "    # allowing some up and down tolerance\n",
    "    if (\"top\" in curr.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "            adj1 = adj\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"top\" in adj.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = curr\n",
    "            adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"bottom\" in curr.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "            adj1 = adj\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"bottom\"in adj.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = curr\n",
    "            adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "\n",
    "                            \n",
    "# IMPORTANT: enrich_factor determines how many \"duplications\" of TRUE values will we have in the train set\n",
    "# This allows for a more balanced train set however, it reduces the strictness of the matches \n",
    "# i.e. (not sure why) when we have multiple nearby \"duplicates\" matches we get much more matches in the validation\n",
    "# PARAMS: enrich_factor=1 means no enrich/duplicate, 20 means duplicate by 20, every 10 pixels\n",
    "# PARAMS: tolerance_factor=0 means only match against exact horizon, each notch equals additional 10 pixels tolerance\n",
    "def NEW_build_train_set_for_binary_labeling(cubes, adjs, slice_size, folder, enrich_factor=2, tolerance_factor=0): \n",
    "    # enrich_factor is split by 2 because it is dual-sided and 1 means actually no enrichment - i.e. 0.5\n",
    "    enrich_factor = enrich_factor / 2 \n",
    "    # clean folder before starting\n",
    "\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    is_enriched = []\n",
    "    discard_c = 0\n",
    "    skipped_c = 0\n",
    "    curr_idx = 0\n",
    "\n",
    "    # import pdb; pdb.set_trace()\n",
    "    # iterate over the cubes\n",
    "    for curr in cubes:\n",
    "        curr_idx = curr_idx + 1\n",
    "        if curr_idx % 100 == 0:\n",
    "            print(\"*** CUBES=\"+str(curr_idx))\n",
    "            print(\" MATCHED=\"+str(sum(x[1] == 1 for x in train_lbls)))\n",
    "            print(\" NOT MATCHED=\"+str(sum(x[0] == 1 for x in train_lbls)))\n",
    "            print(\" DISCARDED=\"+str(discard_c))\n",
    "            print(\" SKIPPED=\"+str(skipped_c))\n",
    "            \n",
    "        # iterate over the others (effectively n^2)\n",
    "        for adj in adjs:\n",
    "            # Initial filter: what CAN be matched against what?\n",
    "            # 1 - not of the same fragment (file==fragment)\n",
    "            # 2 - they ARE of the same tear - don't want to confuse the learning with false data coming from different tears\n",
    "            # 3 - no need to test against self and avoid checking from both directions\n",
    "            if  adj[\"file\"] != curr[\"file\"] and \\\n",
    "                adj[\"tear\"] == curr[\"tear\"] and \\\n",
    "                curr[\"piece_col\"] < adj[\"piece_col\"]: \n",
    "                # last condition above - actually ignores pieces of the same col but different rows\n",
    "                # the assumption is that they are either \"not-match\" and then will tilt the balance further to not-match\n",
    "                # or they are \"somewhat-matching\" but in a way that might confuse the algorithm\n",
    "                # import pdb; pdb.set_trace()\n",
    "                \n",
    "                # print(\">>> >>>\"+str(curr[\"cube\"].shape)+\" <<< <<<\"+str(adj[\"cube\"].shape))\n",
    "                # append the adjacent image to the current image\n",
    "                conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr, adj, slice_size)\n",
    "\n",
    "                train_x_delta.append(x_delta)\n",
    "                train_y_delta.append(y_delta)\n",
    "                                \n",
    "                # Condition for marking as match:\n",
    "                # 1 - the adj piece is on the same row as the curr\n",
    "                # 2 - the adj piece is just to the right of the curr\n",
    "                # 3 - the curr cube is on the right edge of the piece\n",
    "                # 4 - the adj cube is on the left edge of the piece\n",
    "                # 5 - the cubes are in the same horizon\n",
    "                if  curr[\"piece_row\"] == adj[\"piece_row\"] and \\\n",
    "                    curr[\"piece_col\"] + 1 == adj[\"piece_col\"] and \\\n",
    "                    (curr[\"col\"] != 0 or curr[\"last\"]) and \\\n",
    "                    (adj[\"col\"] == 0 or not adj[\"last\"]) and \\\n",
    "                    np.abs(x_delta) < 50: \n",
    "\n",
    "                    #print(\"TTT:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #print(\"TTT:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    \n",
    "                    # tag the image as matched (2nd digit) and not-enriched (1st digit)\n",
    "                    output = folder+\"0_1=\"+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                    # print(\">>> MATCH >>>\"+output)\n",
    "                    save_img(output, conc)\n",
    "                    # print(\">>> >>> >>> SAVED\")\n",
    "                    train_imgs.append(output)\n",
    "                    train_lbls.append([0,1])\n",
    "                    is_enriched.append(False)\n",
    "\n",
    "                    # TOLERANCE\n",
    "                    VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                              train_y_delta, is_enriched, curr, adj, tolerance_factor)\n",
    "                    \n",
    "                    # ENRICH/DUPLICATE\n",
    "                    # need to enrich the set with a few more true positive samples - so we offset \n",
    "                    # the matched images up and down a few times and create more matches\n",
    "                    if (\"top\" in curr.keys() and \"top\" in adj.keys()):\n",
    "                        for i in range(0, 121, int(120/enrich_factor)):\n",
    "                            if i == 0:\n",
    "                                continue\n",
    "                            curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "                            adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "                            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "                            # tag the image as matched (2nd digit) and enriched (1st digit)\n",
    "                            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "                            # print(\">>> MATCH >>>\"+output)\n",
    "                            save_img(output, conc)\n",
    "                            # print(\">>> >>> >>> SAVED\")\n",
    "                            train_imgs.append(output)\n",
    "                            train_x_delta.append(x_delta)\n",
    "                            train_y_delta.append(y_delta)\n",
    "                            # mark the image as matched\n",
    "                            train_lbls.append([0,1])\n",
    "                            is_enriched.append(True)\n",
    "\n",
    "                            # TOLERANCE\n",
    "                            VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                                      train_y_delta, is_enriched, curr1, adj1, tolerance_factor)\n",
    "                            \n",
    "                    if (\"bottom\" in curr.keys() and \"bottom\"in adj.keys()):\n",
    "                        for i in range(0, 121, int(120/enrich_factor)):\n",
    "                            if i == 0:\n",
    "                                continue\n",
    "                            curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "                            adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "                            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "                            # tag the image as matched (2nd digit) and enriched (1st digit)\n",
    "                            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "                            # print(\">>> MATCH >>>\"+output)\n",
    "                            save_img(output, conc)\n",
    "                            # print(\">>> >>> >>> SAVED\")\n",
    "                            train_imgs.append(output)\n",
    "                            train_x_delta.append(x_delta)\n",
    "                            train_y_delta.append(y_delta)\n",
    "                            # mark the image as matched\n",
    "                            train_lbls.append([0,1])\n",
    "                            is_enriched.append(True)\n",
    "\n",
    "                            # TOLERANCE\n",
    "                            VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                                      train_y_delta, is_enriched, curr1, adj1, tolerance_factor)\n",
    " \n",
    "                # adding a condition for marking as not-matched - we mark only the \"key\" cubes which are every 250px\n",
    "                # and not overlap - hence we reduce the ratio in favour of not matched which is enormous\n",
    "                \n",
    "                # altering between the next 2 lines allows to control the number/ratio of non-match:match\n",
    "                # elif int(curr[\"row\"]) == curr[\"row\"] and int(adj[\"row\"]) == adj[\"row\"]: # this condition will match curr key cubes with adj key cubes only\n",
    "                elif int(adj[\"row\"]) == adj[\"row\"]: # this condition will allow curr cubes which are not just key\n",
    "                    #print(\"NNN:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #print(\"NNN:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    # tag the image as not-matched (2nd digit) and not-enriched (1st digit)\n",
    "                    output = folder+\"0_0=\"+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                    # print(\"<<< nonmatch <<<\"+output)\n",
    "                    save_img(output, conc)\n",
    "                    # print(\"<<< <<< <<< SAVED\")\n",
    "                    train_imgs.append(output)\n",
    "                    train_lbls.append([1,0]) # not matched\n",
    "                    is_enriched.append(False)\n",
    "                \n",
    "                # discard not matched which are not \"key\" cubes (every 250px)\n",
    "                else:\n",
    "                    #if x_delta == 0:\n",
    "                    #    print(\"DDD:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #    print(\"DDD:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    discard_c += 1\n",
    "            else:\n",
    "                skipped_c += 1\n",
    "    print(\"*** MATCHED=\"+str(sum(x[1] == 1 for x in train_lbls)))\n",
    "    print(\"*** NOT MATCHED=\"+str(sum(x[0] == 1 for x in train_lbls)))\n",
    "    print(\"*** DISCARDED=\"+str(discard_c))\n",
    "    print(\"*** SKIPPED=\"+str(skipped_c))\n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched\n",
    "\n",
    "\n",
    "def PARALLEL_build_train_set_for_binary_labeling(args): \n",
    "    print('args:', args)\n",
    "    return NEW_build_train_set_for_binary_labeling(args[0], args[1], args[2], args[3], args[4], args[5])\n",
    "\n",
    "\n",
    "def frame_to_n_by_m(orig, start_vector, end_vector, is_col):\n",
    "    max_val = np.amax(end_vector)\n",
    "    min_val = np.amin(start_vector)\n",
    "    width = max_val - min_val\n",
    "    if width < CUBE_SIZE:\n",
    "        width = CUBE_SIZE\n",
    "    if (is_col):\n",
    "        result = np.zeros((start_vector.size, width))\n",
    "    else:\n",
    "        result = np.zeros((width, start_vector.size))\n",
    "    \n",
    "    for i in range(0, start_vector.size):\n",
    "        if (is_col):\n",
    "            row_vec = orig[i, start_vector[i]:end_vector[i]]\n",
    "        else:\n",
    "            row_vec = orig[start_vector[i]:end_vector[i],i]\n",
    "        temp = np.lib.pad(row_vec, (start_vector[i]-min_val, max_val-end_vector[i]), 'constant', constant_values=(0.09, 0.09))\n",
    "        if (is_col):\n",
    "            if (result[i].size != width):\n",
    "                import pdb; pdb.set_trace()\n",
    "            result[i] = temp[0:width]\n",
    "        else:\n",
    "            result[:,i] = temp[0:width]\n",
    "    return min_val, result\n",
    "\n",
    "def rough_tear_line(orig, start_vector, cut_mean, is_col, chew_factor):\n",
    "    end_vector = np.empty(start_vector.size).astype(int)\n",
    "    if (is_col and np.absolute(cut_mean-orig.shape[1]) < 10):\n",
    "        end_vector.fill(orig.shape[1])\n",
    "    elif (not is_col and np.absolute(cut_mean-orig.shape[0]) < 10):\n",
    "        end_vector.fill(orig.shape[0])\n",
    "    else:\n",
    "        deviation_vector = np.random.normal(0, chew_factor, start_vector.size).astype(int)\n",
    "        end_vector[0] = cut_mean + deviation_vector[0]\n",
    "        for i in range(1, end_vector.size):\n",
    "            end_vector[i] = end_vector[i - 1] + deviation_vector[i]\n",
    "\n",
    "        cut_max = start_vector + (CUBE_SIZE + EDGE_GAP)\n",
    "        max_and_end = np.append([end_vector], [cut_max], axis=0)\n",
    "        end_vector = np.amax(max_and_end, axis=0)\n",
    "            \n",
    "    start_px, cut_piece = frame_to_n_by_m(orig, start_vector, end_vector, is_col)    \n",
    "    return start_px, cut_piece, end_vector\n",
    "\n",
    "def rough_tear_image(image, cols, rows):\n",
    "    pieces = []\n",
    "    col_width = int(image.shape[1] / cols)\n",
    "    row_height = int(image.shape[0] / rows)\n",
    "    # print(col_width, row_height)\n",
    "    next_col_start_vec = np.zeros((image.shape[0],), dtype=int)\n",
    "    for col_idx in range(0, cols):\n",
    "    #         import pdb; pdb.set_trace()\n",
    "        start_col_px, cut_column, next_col_start_vec =  rough_tear_line(image, next_col_start_vec, col_width * (col_idx + 1), True, 3)\n",
    "        next_row_start_vec = np.zeros((cut_column.shape[1],), dtype=int)\n",
    "        for row_idx in range(0, rows):\n",
    "            start_row_px, cut_piece, next_row_start_vec = rough_tear_line(cut_column, next_row_start_vec, row_height * (row_idx + 1), False, 1)\n",
    "\n",
    "            ymin, ymax, xmin, xmax = calc_min_max_coordinates_dynamic(cut_piece, cutoff=BG_2_OBJ_RATIO)\n",
    "            temp = crop(cut_piece, ymin, ymax, xmin, xmax)\n",
    "            \n",
    "            #import pdb; pdb.set_trace()\n",
    "            piece = {}\n",
    "            piece[\"orig\"] = cut_piece\n",
    "            piece[\"cut\"] = temp\n",
    "            piece[\"col\"] = col_idx\n",
    "            piece[\"row\"] = row_idx\n",
    "            piece[\"col_px\"] = start_col_px + xmin\n",
    "            piece[\"row_px\"] = start_row_px + ymin\n",
    "            pieces.append(piece)\n",
    "            \n",
    "    return pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.628035Z",
     "start_time": "2019-10-30T20:58:15.621098Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Define model util functions\n",
    "\n",
    "# initialize a shaped matrix of weights with random values\n",
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "# initialize a shaped matrix of bias with random values\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "def max_pool_1x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 1, 2, 1],\n",
    "                        strides=[1, 1, 2, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x1(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 1, 1],\n",
    "                        strides=[1, 2, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_1x1(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 1, 1, 1],\n",
    "                        strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_5x5(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 5, 5, 1],\n",
    "                        strides=[1, 5, 5, 1], padding='SAME')\n",
    "\n",
    "def max_pool_5x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 5, 2, 1],\n",
    "                        strides=[1, 5, 2, 1], padding='SAME')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.649293Z",
     "start_time": "2019-10-30T20:58:15.630548Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Image utility functions (external source)\n",
    "def branchedPoints(skel):\n",
    "    branch1=np.array([[2, 1, 2], [1, 1, 1], [2, 2, 2]])\n",
    "    branch2=np.array([[1, 2, 1], [2, 1, 2], [1, 2, 1]])\n",
    "    branch3=np.array([[1, 2, 1], [2, 1, 2], [1, 2, 2]])\n",
    "    branch4=np.array([[2, 1, 2], [1, 1, 2], [2, 1, 2]])\n",
    "    branch5=np.array([[1, 2, 2], [2, 1, 2], [1, 2, 1]])\n",
    "    branch6=np.array([[2, 2, 2], [1, 1, 1], [2, 1, 2]])\n",
    "    branch7=np.array([[2, 2, 1], [2, 1, 2], [1, 2, 1]])\n",
    "    branch8=np.array([[2, 1, 2], [2, 1, 1], [2, 1, 2]])\n",
    "    branch9=np.array([[1, 2, 1], [2, 1, 2], [2, 2, 1]])\n",
    "    br1=mh.morph.hitmiss(skel,branch1)\n",
    "    br2=mh.morph.hitmiss(skel,branch2)\n",
    "    br3=mh.morph.hitmiss(skel,branch3)\n",
    "    br4=mh.morph.hitmiss(skel,branch4)\n",
    "    br5=mh.morph.hitmiss(skel,branch5)\n",
    "    br6=mh.morph.hitmiss(skel,branch6)\n",
    "    br7=mh.morph.hitmiss(skel,branch7)\n",
    "    br8=mh.morph.hitmiss(skel,branch8)\n",
    "    br9=mh.morph.hitmiss(skel,branch9)\n",
    "    return br1+br2+br3+br4+br5+br6+br7+br8+br9\n",
    "\n",
    "def endPoints(skel):\n",
    "    endpoint1=np.array([[0, 0, 0],\n",
    "                        [0, 1, 0],\n",
    "                        [2, 1, 2]])\n",
    "    \n",
    "    endpoint2=np.array([[0, 0, 0],\n",
    "                        [0, 1, 2],\n",
    "                        [0, 2, 1]])\n",
    "    \n",
    "    endpoint3=np.array([[0, 0, 2],\n",
    "                        [0, 1, 1],\n",
    "                        [0, 0, 2]])\n",
    "    \n",
    "    endpoint4=np.array([[0, 2, 1],\n",
    "                        [0, 1, 2],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint5=np.array([[2, 1, 2],\n",
    "                        [0, 1, 0],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint6=np.array([[1, 2, 0],\n",
    "                        [2, 1, 0],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint7=np.array([[2, 0, 0],\n",
    "                        [1, 1, 0],\n",
    "                        [2, 0, 0]])\n",
    "    \n",
    "    endpoint8=np.array([[0, 0, 0],\n",
    "                        [2, 1, 0],\n",
    "                        [1, 2, 0]])\n",
    "    \n",
    "    ep1=mh.morph.hitmiss(skel,endpoint1)\n",
    "    ep2=mh.morph.hitmiss(skel,endpoint2)\n",
    "    ep3=mh.morph.hitmiss(skel,endpoint3)\n",
    "    ep4=mh.morph.hitmiss(skel,endpoint4)\n",
    "    ep5=mh.morph.hitmiss(skel,endpoint5)\n",
    "    ep6=mh.morph.hitmiss(skel,endpoint6)\n",
    "    ep7=mh.morph.hitmiss(skel,endpoint7)\n",
    "    ep8=mh.morph.hitmiss(skel,endpoint8)\n",
    "    ep = ep1+ep2+ep3+ep4+ep5+ep6+ep7+ep8\n",
    "    return ep\n",
    "\n",
    "def pruning(skeleton, size):\n",
    "    '''remove iteratively end points \"size\" \n",
    "       times from the skeleton\n",
    "    '''\n",
    "    for i in range(0, size):\n",
    "        endpoints = endPoints(skeleton)\n",
    "        endpoints = np.logical_not(endpoints)\n",
    "        skeleton = np.logical_and(skeleton,endpoints)\n",
    "    return skeleton\n",
    "\n",
    "def plot_comparison(original, filtered, filter_name):\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(8, 4), sharex=True, sharey=True)\n",
    "    ax1.imshow(original, cmap=plt.cm.gray)\n",
    "    ax1.set_title('original')\n",
    "    ax1.axis('off')\n",
    "    ax1.set_adjustable('box-forced')\n",
    "    ax2.imshow(filtered, cmap=plt.cm.gray)\n",
    "    ax2.set_title(filter_name)\n",
    "    ax2.axis('off')\n",
    "    ax2.set_adjustable('box-forced')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.665310Z",
     "start_time": "2019-10-30T20:58:15.650750Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_deep - Define the model - 250, 125, 62, 25\n",
    "def model_tf_deep(input_width, forced_bias=0): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob, probability, probabilities #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # zero convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv0 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv0 = bias_variable([32])\n",
    "    h_conv0 = tf.nn.relu(conv2d(x_image, W_conv0) + b_conv0, \"0020\")\n",
    "    h_pool0 = max_pool_1x1(h_conv0) # size is maintained\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv1 = weight_variable([5, 5, 32, 32])\n",
    "#     W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1, \"0021\")\n",
    "#     h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv2 = bias_variable([32])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv3 = bias_variable([32])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*32]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*32, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([1024, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    c_fc2 = tf.constant([0, forced_bias], dtype=tf.float32)\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 + c_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "      # tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y_conv))\n",
    "\n",
    "    probability = tf.nn.softmax(y_conv,1)\n",
    "    \n",
    "    probabilities=tf.reduce_sum(probability,1)\n",
    "    \n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.686052Z",
     "start_time": "2019-10-30T20:58:15.666722Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_deep_tb WITH tensorBoard!! - Define the model - 250, 125, 62, 25\n",
    "def model_tf_deep_tb(input_width, forced_bias=0): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob, probability, probabilities, merged_summary #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    with tf.name_scope('input'):\n",
    "        x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "        x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # zero convolutional layer: one input image and 32 output filters of 5x5\n",
    "    with tf.name_scope('conv_0'):\n",
    "        W_conv0 = weight_variable([5, 5, 1, 32])\n",
    "        b_conv0 = bias_variable([32])\n",
    "        h_conv0 = tf.nn.relu(conv2d(x_image, W_conv0) + b_conv0, \"0020\")\n",
    "        h_pool0 = max_pool_1x1(h_conv0) # size is maintained\n",
    "        tf.summary.histogram(\"weights\", W_conv0)\n",
    "        tf.summary.histogram(\"biases\", b_conv0)\n",
    "        tf.summary.histogram(\"activation\", h_conv0)\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "    with tf.name_scope('conv_1'):\n",
    "        W_conv1 = weight_variable([5, 5, 32, 32])\n",
    "    #     W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "        b_conv1 = bias_variable([32])\n",
    "        h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1, \"0021\")\n",
    "    #     h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "        if (input_width == 250):\n",
    "            h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "        elif (input_width == 125):\n",
    "            h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "        elif (input_width == 62):\n",
    "            h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "        elif (input_width == 25):\n",
    "            h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "        else:\n",
    "            print(\"ERROR - unsupported slice width\")\n",
    "            return\n",
    "        tf.summary.histogram(\"weights\", W_conv1)\n",
    "        tf.summary.histogram(\"biases\", b_conv1)\n",
    "        tf.summary.histogram(\"activation\", h_conv1)\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    with tf.name_scope('conv_2'):\n",
    "        W_conv2 = weight_variable([5, 5, 32, 32])\n",
    "        b_conv2 = bias_variable([32])\n",
    "        h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "        if (input_width == 62):\n",
    "            h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "        elif (input_width == 25):\n",
    "            h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "        else:\n",
    "            h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "        tf.summary.histogram(\"weights\", W_conv2)\n",
    "        tf.summary.histogram(\"biases\", b_conv2)\n",
    "        tf.summary.histogram(\"activation\", h_conv2)\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    with tf.name_scope('conv_3'):\n",
    "        W_conv3 = weight_variable([5, 5, 32, 32])\n",
    "        b_conv3 = bias_variable([32])\n",
    "        h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "        if (input_width == 25):\n",
    "            h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "        else:\n",
    "            h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "        tf.summary.histogram(\"weights\", W_conv3)\n",
    "        tf.summary.histogram(\"biases\", b_conv3)\n",
    "        tf.summary.histogram(\"activation\", h_conv3)\n",
    "\n",
    "\n",
    "    with tf.name_scope('reshape'):\n",
    "        h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*32]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    with tf.name_scope('fully_conn_1'):\n",
    "        W_fc1 = weight_variable([25*25*32, 1024])\n",
    "        b_fc1 = bias_variable([1024])\n",
    "        h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "        tf.summary.histogram(\"weights\", W_fc1)\n",
    "        tf.summary.histogram(\"biases\", b_fc1)\n",
    "        tf.summary.histogram(\"activation\", h_fc1)\n",
    "\n",
    "    # a drop layer with probability \n",
    "    with tf.name_scope('drop'):\n",
    "        keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "        h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    with tf.name_scope('fully_connected_2'):\n",
    "        W_fc2 = weight_variable([1024, 2])\n",
    "        b_fc2 = bias_variable([2])\n",
    "        c_fc2 = tf.constant([0, forced_bias], dtype=tf.float32)\n",
    "        y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 + c_fc2\n",
    "        tf.summary.histogram(\"weights\", W_fc2)\n",
    "        tf.summary.histogram(\"biases\", b_fc2)\n",
    "        tf.summary.histogram(\"activation\", y_conv)\n",
    "        \n",
    "    # minimize loss function\n",
    "    with tf.name_scope('loss_func'):\n",
    "        cross_entropy = tf.reduce_mean(\n",
    "          tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "          # tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y_conv))\n",
    "        tf.summary.scalar('cross_entropy', cross_entropy)\n",
    "        \n",
    "    with tf.name_scope('softmax'):\n",
    "        probability = tf.nn.softmax(y_conv,1)\n",
    "        probabilities=tf.reduce_sum(probability,1)\n",
    "    \n",
    "    # define train step and rate\n",
    "    with tf.name_scope('optimize'):\n",
    "        train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    with tf.name_scope('prediction'):\n",
    "        correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        tf.summary.scalar('accuracy', accuracy)\n",
    "    \n",
    "    merged_summary = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.698913Z",
     "start_time": "2019-10-30T20:58:15.687250Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_orig - Define the model - 250, 125, 62, 25\n",
    "def model_tf_orig(input_width): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # zero convolutional layer: one input image and 32 output filters of 5x5\n",
    "#     W_conv0 = weight_variable([5, 5, 1, 32])\n",
    "#     b_conv0 = bias_variable([32])\n",
    "#     h_conv0 = tf.nn.relu(conv2d(x_image, W_conv0) + b_conv0, \"0020\")\n",
    "#     h_pool0 = max_pool_1x1(h_conv0) # size is maintained\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "#     W_conv1 = weight_variable([5, 5, 32, 32])\n",
    "    W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "#     h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1, \"0021\")\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv2 = bias_variable([32])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv3 = bias_variable([32])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*32]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*32, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([1024, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.713493Z",
     "start_time": "2019-10-30T20:58:15.700417Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_wide - Define the model - 250, 125, 62, 25\n",
    "def model_tf_wide(input_width): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv1 = weight_variable([5, 5, 1, 64])\n",
    "    b_conv1 = bias_variable([64])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 64, 64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 64, 64])\n",
    "    b_conv3 = bias_variable([64])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*64]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*64, 2048])\n",
    "    b_fc1 = bias_variable([2048])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([2048, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.724353Z",
     "start_time": "2019-10-30T20:58:15.714914Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN train\n",
    "def train(train_imgs, train_lbls, output_model, input_model=\"\"):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"TRAINING:\")\n",
    "    print(\"MODEL:\"+output_model)\n",
    "    print(\"#####################################################################\")\n",
    "\n",
    "    from random import randrange\n",
    "    \n",
    "    # TRAIN Prepare the session\n",
    "\n",
    "    # create a saver object\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "    \n",
    "    writer.add_graph(sess.graph)\n",
    "    \n",
    "    if input_model != \"\":\n",
    "        # Restore variables from disk.\n",
    "        saver.restore(sess, input_model)\n",
    "        print(\"Model restored.\")\n",
    "    else:\n",
    "        sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    # TRAIN Train the model\n",
    "    x_batch = []\n",
    "    y_batch = []\n",
    "    # run the train batches\n",
    "    for i in range(BATCHES):\n",
    "        x_batch = []\n",
    "        y_batch = []\n",
    "        for _ in range(BATCH_SIZE):\n",
    "            random_index = randrange(0,len(train_imgs))\n",
    "            image = np.load(train_imgs[random_index]+\".npy\")\n",
    "            # print(train_imgs[random_index])\n",
    "            x_batch.append(image)\n",
    "            y_batch.append(train_lbls[random_index])\n",
    "\n",
    "        # train\n",
    "        # print(\"step %d\"%(i))\n",
    "        train_step.run(feed_dict={x: x_batch, y_: y_batch, keep_prob: 0.5})\n",
    "\n",
    "        # print the accuracy thus far\n",
    "        if (i+1)%50 == 0:\n",
    "            train_accuracy = accuracy.eval(feed_dict={\n",
    "                x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "            print(\"step %d, training accuracy %.2f\"%(i, train_accuracy))\n",
    "            \n",
    "            s=sess.run(merged_summary, feed_dict={x: x_batch, y_: y_batch, keep_prob: 0.5})\n",
    "            writer.add_summary(s, i)\n",
    "\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    train_accuracy = accuracy.eval(feed_dict={\n",
    "        x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "    print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "\n",
    "    # Save the variables to disk.\n",
    "    save_path = saver.save(sess, output_model)\n",
    "    print(\"Model saved in file: %s\" % save_path)\n",
    "\n",
    "    # Close the Session when we're done. If un-commented - need to run next bock of restore...\n",
    "    sess.close()   \n",
    "    print(\"#####################################################################\")\n",
    "    print(\"TRAINING ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.730133Z",
     "start_time": "2019-10-30T20:58:15.725725Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN pre_process - OLD?\n",
    "def pre_process(folder):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS:\"+folder)\n",
    "    print(\"#####################################################################\")\n",
    "    result = []\n",
    "    \n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for file_ in files:\n",
    "            # Read the image\n",
    "            # image = img.imread(os.path.join(root, file_))\n",
    "            image = np.load(os.path.join(root, file_))\n",
    "            # import pdb; pdb.set_trace()\n",
    "            cubes = VAL_slice_to_static_slices(file_, image)\n",
    "            print(\"File: %s >>> cubes: %d\"%(file_, len(cubes)))\n",
    "            result.extend(cubes)\n",
    "    \n",
    "    return result\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.740569Z",
     "start_time": "2019-10-30T20:58:15.731443Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN pre_process_training - crop image, then tear it randomly to various tears, then per tear create cubes out of the edges, return cube set\n",
    "def pre_process_training(img_name, x_start=X_START, x_end=X_END, y_start=Y_START, y_end=Y_END, max_cols=8, max_rows=4):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS:\"+img_name)\n",
    "    print(\"#####################################################################\")\n",
    "    short_name = img_name[:img_name.rfind('-D')]\n",
    "    image = read_and_crop(img_name, x_start, x_end, y_start, y_end)\n",
    "    result = []\n",
    "    \n",
    "    for root, dirs, files in os.walk(ROOT_FOLDER+fragments_folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    \n",
    "    tot_pieces = 0\n",
    "    for col_cut in range(3, max_cols): # 9 3...10\n",
    "        for row_cut in range(2, max_rows): # 6 2...5\n",
    "            print(\"PRE_PROCESS:::\"+\"TEAR_\"+str(col_cut)+\"X\"+str(row_cut))\n",
    "            pieces = rough_tear_image(image, col_cut, row_cut)\n",
    "            tot_pieces += len(pieces)\n",
    "            \n",
    "            for piece in pieces:\n",
    "                # print(\"PRE_PROCESS:::\"+\"PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"]))\n",
    "                fragment_name = short_name + \"_TEAR_\"+str(col_cut)+\"X\"+str(row_cut)+\"_PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "                fragment_file_name = short_name + \"_\"+str(col_cut)+\"X\"+str(row_cut)+\"_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "                # import pdb; pdb.set_trace()\n",
    "                plt.imsave(os.path.join(ROOT_FOLDER+fragments_folder,fragment_file_name+\".jpg\"), piece[\"cut\"], cmap=plt.cm.gray)\n",
    "                cubes = VAL_slice_TEAR_to_static_slices(fragment_name, piece)\n",
    "                for cube in cubes:\n",
    "                    cube[\"tear\"] = str(col_cut)+\"X\"+str(row_cut)\n",
    "                    cube[\"piece_col\"] = piece[\"col\"]\n",
    "                    cube[\"piece_row\"] = piece[\"row\"]\n",
    "                #print(\"Pieces: %d >>> cubes: %d\"%(len(pieces), len(cubes)))\n",
    "                result.extend(cubes)\n",
    "    \n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS ENDED\")\n",
    "    print(\"Pieces: %d >>> cubes: %d\"%(tot_pieces, len(result)))\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.745376Z",
     "start_time": "2019-10-30T20:58:15.742161Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def validate1(cubes, model, slice_size, folder, curr_cube):    \n",
    "    # VALIDATE prepare the data sets\n",
    "    test_imgs, test_x_delta, test_y_delta, test_x_file, test_y_file = VAL_build_train_set(cubes, slice_size, folder, curr_cube)\n",
    "    print(\"loaded %d images\"%(len(test_imgs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.758162Z",
     "start_time": "2019-10-30T20:58:15.746814Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def validate2(folder, model, slice_size):\n",
    "    test_imgs = []\n",
    "    test_x_file = []\n",
    "    test_y_file = []\n",
    "    the_root = \"\"\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        the_root = root\n",
    "        for file_ in files:\n",
    "            test_imgs.append( os.path.join(root, file_) )\n",
    "            test_x_file.append(file_[:file_.rfind('---P')])\n",
    "            test_y_file.append(file_[file_.rfind('---P')+3:])\n",
    "            \n",
    "    print(len(test_imgs))\n",
    "    \n",
    "    # VALIDATE Prepare a test session \n",
    "\n",
    "    # Add ops to save and restore all the variables.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    # Restore variables from disk.\n",
    "    saver.restore(sess, model)\n",
    "    print(\"Model restored.\")\n",
    "\n",
    "    # VALIDATE Validate the model\n",
    "\n",
    "    # import pdb; pdb.set_trace()\n",
    "    v1t = []\n",
    "    count = 0\n",
    "    length = len(test_imgs)\n",
    "    batch = 100\n",
    "    x_batch = []\n",
    "    # change the ranges in the loop below - first number is the start point (multiplied by batch size)\n",
    "    # second number is the end point (multiplied by batch size)\n",
    "    # third number is the jump from batch to batch\n",
    "    # use the length about to set the batch length\n",
    "    for start in range(0, length, batch):\n",
    "        for i in range(start, start+batch):\n",
    "            if (i < length):\n",
    "                image = np.load(test_imgs[i])\n",
    "                x_batch.append(image)\n",
    "                count += 1\n",
    "\n",
    "        # print(\"Validating start at #%d end at %d\"%(start*batch,(start+length)*batch))\n",
    "        my_prediction=tf.argmax(y_conv,1)\n",
    "        v1 = my_prediction.eval(feed_dict={x:x_batch, keep_prob: 1.0})\n",
    "        v1t = np.concatenate((v1t, v1), axis=0)\n",
    "        x_batch = []\n",
    "        print(\">>> step %d\"%(start+batch))\n",
    "\n",
    "\n",
    "    match_indexes = np.nonzero(v1t)[0]\n",
    "    A = np.array(test_x_file)\n",
    "    B = np.array(test_y_file)\n",
    "    C = np.array(test_imgs)\n",
    "    match_x_files = A[match_indexes]\n",
    "    match_y_files = B[match_indexes]\n",
    "    match_images = C[match_indexes]\n",
    "    \n",
    "    for matched_img in match_images:\n",
    "        load_img = np.load(matched_img)\n",
    "        plt.imsave(os.path.join(\"/Volumes/250GB/matched/\",matched_img[matched_img.rfind('/')+1:]+\".png\"), load_img, cmap=plt.cm.gray)\n",
    "    \n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for file_ in files:\n",
    "            os.remove( os.path.join(root, file_) ) # delete it from the FS\n",
    "                \n",
    "    with open('matches.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for match_index in match_indexes:\n",
    "            print(\"MATCH %s === %s\"%(test_x_file[match_index], test_y_file[match_index]))\n",
    "            # print(\"MATCH %s === %s\"%(A[match_index], B[match_index]))\n",
    "            # csvout.writerow([A[match_index], B[match_index]])\n",
    "            csvout.writerow([test_x_file[match_index], test_y_file[match_index]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    # Close the Session when we're done.\n",
    "    sess.close()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.774377Z",
     "start_time": "2019-10-30T20:58:15.759743Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def validate2_for_cross_validation(test_imgs, test_lbls, is_enriched, model, max_samples=0):\n",
    "    print(len(test_imgs))\n",
    "    \n",
    "    # VALIDATE Prepare a test session \n",
    "\n",
    "    # Add ops to save and restore all the variables.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    # Restore variables from disk.\n",
    "    saver.restore(sess, model)\n",
    "    print(\"Model restored.\")\n",
    "\n",
    "    # VALIDATE Validate the model\n",
    "\n",
    "    count = 0\n",
    "    se = 0\n",
    "    st = 0\n",
    "    v1t = []\n",
    "    v2t = []\n",
    "    v1tt = []\n",
    "    v2tt = []\n",
    "    length = len(test_imgs)\n",
    "    if max_samples != 0:\n",
    "        length = max_samples\n",
    "    batch = 100\n",
    "    x_batch = []\n",
    "    y_batch = []\n",
    "    # change the ranges in the loop below - first number is the start point (multiplied by batch size)\n",
    "    # second number is the end point (multiplied by batch size)\n",
    "    # third number is the jump from batch to batch\n",
    "    # use the length about to set the batch length\n",
    "    for start in range(0, length, batch):\n",
    "        for i in range(start, start+batch):\n",
    "            if (i < length):\n",
    "                image = np.load(test_imgs[i]+\".npy\")\n",
    "                x_batch.append(image)\n",
    "                y_batch.append(train_lbls[i])                \n",
    "                \n",
    "        # print the accuracy thus far\n",
    "    #         train_accuracy = accuracy.eval(feed_dict={\n",
    "    #             x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "    #         print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "\n",
    "        # print(\"Validating start at #%d end at %d\"%(start*batch,(start+length)*batch))\n",
    "    #         my_prediction=tf.argmax(y_conv,1)\n",
    "    #         v1 = my_prediction.eval(feed_dict={x:x_batch, keep_prob: 1.0})\n",
    "    #         v1t = np.concatenate((v1t, v1), axis=0)\n",
    "\n",
    "        ######## printing the predictions and their normalized values\n",
    "        # print(\"y_conv=\"+str(y_conv.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        # print(\"probability=\"+str(probability.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        # print(\"probabilities=\"+str(probabilities.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        \n",
    "        my_prediction=tf.argmax(y_conv,1)\n",
    "        my_target=tf.argmax(y_,1)\n",
    "        v1 = my_prediction.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "        v2 = my_target.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "        v1t = np.concatenate((v1t, v1), axis=0)\n",
    "        v2t = np.concatenate((v2t, v2), axis=0)\n",
    "\n",
    "        c1 = np.sum(np.absolute(np.subtract(v2, v1)))\n",
    "        c2 = np.sum(np.absolute(v2))\n",
    "        se += c1\n",
    "        st += c2\n",
    "\n",
    "        x_batch = []\n",
    "        y_batch = []\n",
    "        print(\">>> step %d\"%(start+batch))\n",
    "        \n",
    "        count += ((i+1) - start)\n",
    "        precision, recall, f_score, support = precision_recall_fscore_support(v2t, v1t, average='binary')\n",
    "        print(\"step %d-%d, precision %f, recall %f, f_score %f\"%(start, i, precision, recall, f_score))\n",
    "        # print(\"Accumulated total true = %d\"%(st));\n",
    "        # print(\"Accumulated total error rate = %f\"%(se/count));\n",
    "        # v1tt = np.concatenate((v1tt, v1t), axis=0)\n",
    "        # v2tt = np.concatenate((v2tt, v2t), axis=0)\n",
    "        print(\"=== total %d match %d\"%(count, len(np.nonzero(v1t)[0])))\n",
    "\n",
    "    precision, recall, f_score, support = precision_recall_fscore_support(v2t, v1t, average='binary')\n",
    "    print(\"TOTAL %d, precision %f, recall %f, f_score %f\"%(count, precision, recall, f_score))\n",
    "    print(\"TOTAL true = %d\"%(st));\n",
    "    print(\"TOTAL error rate = %f\"%(se/count));\n",
    "    \n",
    "    match_indexes = np.nonzero(v1t)[0]\n",
    "    C = np.array(test_imgs)\n",
    "    match_images = C[match_indexes]\n",
    "    \n",
    "    # for matched_img in match_images:\n",
    "    #    load_img = np.load(matched_img+\".npy\")\n",
    "    #    plt.imsave(os.path.join(ROOT_FOLDER+\"synt_matched/\",matched_img[matched_img.rfind('/')+1:]+\".png\"), load_img, cmap=plt.cm.gray)\n",
    "\n",
    "    with open(strftime(\"%Y%m%d_%H%M%S\", gmtime())+'_synt_all.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for idx, test_img in enumerate(test_imgs):\n",
    "            # print(\"MATCH %s === %s\"%(test_imgs[match_index], train_lbls[match_index]))\n",
    "            match_class = 0\n",
    "            if idx in match_indexes:\n",
    "                match_class = 1\n",
    "            csvout.writerow([test_img, train_lbls[idx], match_class, is_enriched[idx]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    with open(strftime(\"%Y%m%d_%H%M%S\", gmtime())+'_synt_matches.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for match_index in match_indexes:\n",
    "            # print(\"MATCH %s === %s\"%(test_imgs[match_index], train_lbls[match_index]))\n",
    "            csvout.writerow([test_imgs[match_index], train_lbls[match_index], is_enriched[match_index]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    # Close the Session when we're done.\n",
    "    sess.close()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.780176Z",
     "start_time": "2019-10-30T20:58:15.775796Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def iter_validate(cubes, model, slice_size, folder):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"VALIDATING\")\n",
    "    print(\"#####################################################################\")\n",
    "    cubes_len = len(cubes)\n",
    "    batch_size = 100\n",
    "    count = 0\n",
    "    # iterate over the cubes\n",
    "    for curr in cubes:\n",
    "        count += 1\n",
    "        if count < batch_size: ### TEMP LIMITATION\n",
    "            print(\"CUBE:%s\"%(curr[\"file\"]+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])))\n",
    "            validate1(cubes, model, slice_size, folder, curr)\n",
    "            validate2(folder, model, slice_size)\n",
    "        \n",
    "    print(\"#####################################################################\")\n",
    "    print(\"VALIDATION ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.784522Z",
     "start_time": "2019-10-30T20:58:15.781758Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run_all(folder, model, slice_size):\n",
    "    model_tf(slice_size)\n",
    "    cubes_set = pre_process(folder)\n",
    "    validate(cubes_set, model, slice_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.789588Z",
     "start_time": "2019-10-30T20:58:15.787432Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# HELPER block\n",
    "# image = read_and_crop(\"PX303/FG001/PX303-Fg001-V-C01-R01-D05032015-T112602-ML924__012.jpg\")\n",
    "## image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C01-R01-D08032015-T110900-ML924__012.jpg\", 100, -1, 400, -1)\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C01-R02-D08032015-T105147-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C02-R01-D08032015-T110025-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C02-R02-D08032015-T105553-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C01-R01-D08032015-T120605-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C01-R02-D08032015-T115230-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C02-R01-D08032015-T120158-ML924__012.jpg\")\n",
    "##image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C02-R02-D08032015-T115704-ML924__012.jpg\", 0, 6200, 0, 4400)\n",
    "##plt.imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:58:15.796195Z",
     "start_time": "2019-10-30T20:58:15.791539Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def load_train_from_disk(path):\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    is_enriched = []\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file_ in files:\n",
    "            file_name = os.path.join(root, file_)\n",
    "            file_name = file_name[:file_name.rfind(\".\")]\n",
    "            train_imgs.append(file_name)\n",
    "    #        train_lbls.append([1,0] if file_.startswith(\"0=\") else [0,1])\n",
    "    # return train_imgs, train_lbls\n",
    "            enriched = file_[0] == '1'\n",
    "            is_enriched.append(enriched)\n",
    "            label = [0,1] if file_[2] == '1' else [1,0] \n",
    "            train_lbls.append(label)\n",
    "    return train_imgs, train_lbls, is_enriched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-28T13:46:17.900204Z",
     "start_time": "2019-09-28T13:46:17.898568Z"
    }
   },
   "outputs": [],
   "source": [
    "##########################################\n",
    "########################################## RUN ALL ABOVE !!! Then select the relevant code below\n",
    "##########################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T20:10:16.171175Z",
     "start_time": "2019-10-30T19:49:21.180064Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/tb_tear_model1.ckpt\n",
      "#####################################################################\n",
      "step 49, training accuracy 0.80\n",
      "step 99, training accuracy 0.68\n",
      "step 149, training accuracy 0.80\n",
      "step 199, training accuracy 0.62\n",
      "step 249, training accuracy 0.77\n",
      "step 299, training accuracy 0.82\n",
      "step 349, training accuracy 0.83\n",
      "step 399, training accuracy 0.77\n",
      "step 449, training accuracy 0.73\n",
      "step 499, training accuracy 0.88\n",
      "step 549, training accuracy 0.77\n",
      "step 599, training accuracy 0.78\n",
      "step 649, training accuracy 0.78\n",
      "step 699, training accuracy 0.83\n",
      "step 749, training accuracy 0.78\n",
      "step 799, training accuracy 0.90\n",
      "step 849, training accuracy 0.95\n",
      "step 899, training accuracy 0.92\n",
      "step 949, training accuracy 0.90\n",
      "step 999, training accuracy 0.95\n",
      "step 1049, training accuracy 0.93\n",
      "step 1099, training accuracy 0.88\n",
      "step 1149, training accuracy 0.93\n",
      "step 1199, training accuracy 0.93\n",
      "step 1249, training accuracy 0.90\n",
      "step 1299, training accuracy 0.88\n",
      "step 1349, training accuracy 0.85\n",
      "step 1399, training accuracy 0.92\n",
      "step 1449, training accuracy 0.92\n",
      "step 1499, training accuracy 0.95\n",
      "step 1549, training accuracy 0.90\n",
      "step 1599, training accuracy 0.90\n",
      "step 1649, training accuracy 0.92\n",
      "step 1699, training accuracy 0.88\n",
      "step 1749, training accuracy 0.92\n",
      "step 1799, training accuracy 0.93\n",
      "step 1849, training accuracy 0.93\n",
      "step 1899, training accuracy 0.93\n",
      "step 1949, training accuracy 0.93\n",
      "step 1999, training accuracy 0.97\n",
      "Optimization Finished!\n",
      "step 1999, training accuracy 0.966667\n",
      "Model saved in file: /media/1KGB_ILAN/papyrus/models/tb_tear_model1.ckpt\n",
      "#####################################################################\n",
      "TRAINING ENDED\n",
      "#####################################################################\n",
      " \n",
      " \n"
     ]
    }
   ],
   "source": [
    "# RUN1 - take 1st large pieces and train on it\n",
    "# cubes_set = pre_process_training(\"PX303-Fg001-V-C01-R01-D05032015-T112520-ML638__006.jpg\", max_cols=8, max_rows=4)\n",
    "# train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#     NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 12, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep_tb(250)\n",
    "writer = tf.summary.FileWriter(ROOT_FOLDER + \"tf/\" + \"run1\")\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tb_tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T14:21:01.997116Z",
     "start_time": "2019-04-03T14:16:57.383608Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RE-RUN1 - take 1st large pieces and train on it\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats3/\")\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"model_binary/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T06:55:34.462694Z",
     "start_time": "2019-04-01T06:55:34.270110Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# TEMP DEBUG\n",
    "len_cubes = len(cubes_set)\n",
    "print(len_cubes)\n",
    "# bat_cubes = 1000\n",
    "# BATCHES = 500 # smaller train batches\n",
    "# for start in range(0, len_cubes, bat_cubes):\n",
    "#     end = ((start+bat_cubes) if start+bat_cubes < len_cubes else (len_cubes - 1))\n",
    "#     print(\"start:%s end:%s\", start, end)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set[1:2], cubes_set, CUBE_SIZE, \n",
    "                                            ROOT_FOLDER + \"train_concats2/\", 1, 5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-30T22:09:43.194577Z",
     "start_time": "2019-10-30T20:58:24.262091Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "PRE_PROCESS:PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\n",
      "#####################################################################\n",
      "PRE_PROCESS:::TEAR_3X2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to uint16\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE_PROCESS:::TEAR_3X3\n",
      "PRE_PROCESS:::TEAR_4X2\n",
      "PRE_PROCESS:::TEAR_4X3\n",
      "PRE_PROCESS:::TEAR_5X2\n",
      "PRE_PROCESS:::TEAR_5X3\n",
      "PRE_PROCESS:::TEAR_6X2\n",
      "PRE_PROCESS:::TEAR_6X3\n",
      "PRE_PROCESS:::TEAR_7X2\n",
      "PRE_PROCESS:::TEAR_7X3\n",
      "#####################################################################\n",
      "PRE_PROCESS ENDED\n",
      "Pieces: 125 >>> cubes: 8000\n",
      "#####################################################################\n",
      " \n",
      " \n",
      "*** CUBES=100\n",
      " MATCHED=1802\n",
      " NOT MATCHED=6719\n",
      " DISCARDED=26483\n",
      " SKIPPED=758736\n",
      "*** CUBES=200\n",
      " MATCHED=4100\n",
      " NOT MATCHED=12065\n",
      " DISCARDED=47465\n",
      " SKIPPED=1532336\n",
      "*** CUBES=300\n",
      " MATCHED=7533\n",
      " NOT MATCHED=15643\n",
      " DISCARDED=61384\n",
      " SKIPPED=2314736\n",
      "*** CUBES=400\n",
      " MATCHED=8023\n",
      " NOT MATCHED=16181\n",
      " DISCARDED=63470\n",
      " SKIPPED=3112096\n",
      "*** CUBES=500\n",
      " MATCHED=8321\n",
      " NOT MATCHED=16781\n",
      " DISCARDED=65695\n",
      " SKIPPED=3909261\n",
      "*** CUBES=600\n",
      " MATCHED=10093\n",
      " NOT MATCHED=23467\n",
      " DISCARDED=90435\n",
      " SKIPPED=4677761\n",
      "*** CUBES=700\n",
      " MATCHED=12970\n",
      " NOT MATCHED=28473\n",
      " DISCARDED=108898\n",
      " SKIPPED=5454199\n",
      "*** CUBES=800\n",
      " MATCHED=15986\n",
      " NOT MATCHED=31799\n",
      " DISCARDED=121096\n",
      " SKIPPED=6238575\n",
      "*** CUBES=900\n",
      " MATCHED=15986\n",
      " NOT MATCHED=31799\n",
      " DISCARDED=121096\n",
      " SKIPPED=7038575\n",
      "*** CUBES=1000\n",
      " MATCHED=16630\n",
      " NOT MATCHED=35851\n",
      " DISCARDED=136485\n",
      " SKIPPED=7819114\n",
      "*** CUBES=1100\n",
      " MATCHED=19003\n",
      " NOT MATCHED=46234\n",
      " DISCARDED=175927\n",
      " SKIPPED=8569214\n",
      "*** CUBES=1200\n",
      " MATCHED=22939\n",
      " NOT MATCHED=54435\n",
      " DISCARDED=207238\n",
      " SKIPPED=9329582\n",
      "*** CUBES=1300\n",
      " MATCHED=26188\n",
      " NOT MATCHED=61004\n",
      " DISCARDED=232446\n",
      " SKIPPED=10097706\n",
      "*** CUBES=1400\n",
      " MATCHED=29491\n",
      " NOT MATCHED=64585\n",
      " DISCARDED=246366\n",
      " SKIPPED=10880106\n",
      "*** CUBES=1500\n",
      " MATCHED=31341\n",
      " NOT MATCHED=66553\n",
      " DISCARDED=254022\n",
      " SKIPPED=11670426\n",
      "*** CUBES=1600\n",
      " MATCHED=31341\n",
      " NOT MATCHED=66553\n",
      " DISCARDED=254022\n",
      " SKIPPED=12470426\n",
      "*** CUBES=1700\n",
      " MATCHED=32148\n",
      " NOT MATCHED=73448\n",
      " DISCARDED=279248\n",
      " SKIPPED=13238272\n",
      "*** CUBES=1800\n",
      " MATCHED=34857\n",
      " NOT MATCHED=83316\n",
      " DISCARDED=315351\n",
      " SKIPPED=13992208\n",
      "*** CUBES=1900\n",
      " MATCHED=38091\n",
      " NOT MATCHED=90495\n",
      " DISCARDED=341264\n",
      " SKIPPED=14759008\n",
      "*** CUBES=2000\n",
      " MATCHED=41217\n",
      " NOT MATCHED=95155\n",
      " DISCARDED=358362\n",
      " SKIPPED=15537148\n",
      "*** CUBES=2100\n",
      " MATCHED=44026\n",
      " NOT MATCHED=98445\n",
      " DISCARDED=370621\n",
      " SKIPPED=16321508\n",
      "*** CUBES=2200\n",
      " MATCHED=44026\n",
      " NOT MATCHED=98445\n",
      " DISCARDED=370621\n",
      " SKIPPED=17121508\n",
      "*** CUBES=2300\n",
      " MATCHED=44742\n",
      " NOT MATCHED=103607\n",
      " DISCARDED=390399\n",
      " SKIPPED=17896542\n",
      "*** CUBES=2400\n",
      " MATCHED=47440\n",
      " NOT MATCHED=117188\n",
      " DISCARDED=442430\n",
      " SKIPPED=18630842\n",
      "*** CUBES=2500\n",
      " MATCHED=50500\n",
      " NOT MATCHED=128568\n",
      " DISCARDED=486146\n",
      " SKIPPED=19375650\n",
      "*** CUBES=2600\n",
      " MATCHED=52904\n",
      " NOT MATCHED=138270\n",
      " DISCARDED=523559\n",
      " SKIPPED=20128453\n",
      "*** CUBES=2700\n",
      " MATCHED=56511\n",
      " NOT MATCHED=145349\n",
      " DISCARDED=551171\n",
      " SKIPPED=20893653\n",
      "*** CUBES=2800\n",
      " MATCHED=59673\n",
      " NOT MATCHED=150259\n",
      " DISCARDED=570301\n",
      " SKIPPED=21669517\n",
      "*** CUBES=2900\n",
      " MATCHED=63095\n",
      " NOT MATCHED=153838\n",
      " DISCARDED=584218\n",
      " SKIPPED=22451917\n",
      "*** CUBES=3000\n",
      " MATCHED=63429\n",
      " NOT MATCHED=154197\n",
      " DISCARDED=585609\n",
      " SKIPPED=23250157\n",
      "*** CUBES=3100\n",
      " MATCHED=63524\n",
      " NOT MATCHED=156058\n",
      " DISCARDED=592185\n",
      " SKIPPED=24041715\n",
      "*** CUBES=3200\n",
      " MATCHED=65375\n",
      " NOT MATCHED=169339\n",
      " DISCARDED=639129\n",
      " SKIPPED=24781415\n",
      "*** CUBES=3300\n",
      " MATCHED=68234\n",
      " NOT MATCHED=181069\n",
      " DISCARDED=681200\n",
      " SKIPPED=25527515\n",
      "*** CUBES=3400\n",
      " MATCHED=70749\n",
      " NOT MATCHED=190567\n",
      " DISCARDED=715811\n",
      " SKIPPED=26283315\n",
      "*** CUBES=3500\n",
      " MATCHED=73257\n",
      " NOT MATCHED=197646\n",
      " DISCARDED=741142\n",
      " SKIPPED=27050815\n",
      "*** CUBES=3600\n",
      " MATCHED=76565\n",
      " NOT MATCHED=202207\n",
      " DISCARDED=757523\n",
      " SKIPPED=27829763\n",
      "*** CUBES=3700\n",
      " MATCHED=79112\n",
      " NOT MATCHED=205319\n",
      " DISCARDED=768772\n",
      " SKIPPED=28615321\n",
      "*** CUBES=3800\n",
      " MATCHED=79112\n",
      " NOT MATCHED=205319\n",
      " DISCARDED=768772\n",
      " SKIPPED=29415321\n",
      "*** CUBES=3900\n",
      " MATCHED=80367\n",
      " NOT MATCHED=213299\n",
      " DISCARDED=799060\n",
      " SKIPPED=30177016\n",
      "*** CUBES=4000\n",
      " MATCHED=83623\n",
      " NOT MATCHED=230280\n",
      " DISCARDED=863479\n",
      " SKIPPED=30895516\n",
      "*** CUBES=4100\n",
      " MATCHED=85735\n",
      " NOT MATCHED=244897\n",
      " DISCARDED=919388\n",
      " SKIPPED=31624918\n",
      "*** CUBES=4200\n",
      " MATCHED=88629\n",
      " NOT MATCHED=257524\n",
      " DISCARDED=967809\n",
      " SKIPPED=32363778\n",
      "*** CUBES=4300\n",
      " MATCHED=92138\n",
      " NOT MATCHED=268199\n",
      " DISCARDED=1008727\n",
      " SKIPPED=33112078\n",
      "*** CUBES=4400\n",
      " MATCHED=95301\n",
      " NOT MATCHED=276046\n",
      " DISCARDED=1038794\n",
      " SKIPPED=33874067\n",
      "*** CUBES=4500\n",
      " MATCHED=98630\n",
      " NOT MATCHED=282794\n",
      " DISCARDED=1064681\n",
      " SKIPPED=34641331\n",
      "*** CUBES=4600\n",
      " MATCHED=101760\n",
      " NOT MATCHED=286377\n",
      " DISCARDED=1078604\n",
      " SKIPPED=35423731\n",
      "*** CUBES=4700\n",
      " MATCHED=103815\n",
      " NOT MATCHED=288522\n",
      " DISCARDED=1086956\n",
      " SKIPPED=36213171\n",
      "*** CUBES=4800\n",
      " MATCHED=103815\n",
      " NOT MATCHED=288522\n",
      " DISCARDED=1086956\n",
      " SKIPPED=37013171\n",
      "*** CUBES=4900\n",
      " MATCHED=105504\n",
      " NOT MATCHED=299264\n",
      " DISCARDED=1126461\n",
      " SKIPPED=37762867\n",
      "*** CUBES=5000\n",
      " MATCHED=108245\n",
      " NOT MATCHED=315922\n",
      " DISCARDED=1187760\n",
      " SKIPPED=38484815\n",
      "*** CUBES=5100\n",
      " MATCHED=110029\n",
      " NOT MATCHED=329710\n",
      " DISCARDED=1238798\n",
      " SKIPPED=39219915\n",
      "*** CUBES=5200\n",
      " MATCHED=112866\n",
      " NOT MATCHED=341347\n",
      " DISCARDED=1281447\n",
      " SKIPPED=39965534\n",
      "*** CUBES=5300\n",
      " MATCHED=115936\n",
      " NOT MATCHED=351582\n",
      " DISCARDED=1318712\n",
      " SKIPPED=40717934\n",
      "*** CUBES=5400\n",
      " MATCHED=119067\n",
      " NOT MATCHED=358761\n",
      " DISCARDED=1344632\n",
      " SKIPPED=41484734\n",
      "*** CUBES=5500\n",
      " MATCHED=121984\n",
      " NOT MATCHED=364142\n",
      " DISCARDED=1364054\n",
      " SKIPPED=42259834\n",
      "*** CUBES=5600\n",
      " MATCHED=125036\n",
      " NOT MATCHED=367720\n",
      " DISCARDED=1376976\n",
      " SKIPPED=43043234\n",
      "*** CUBES=5700\n",
      " MATCHED=125444\n",
      " NOT MATCHED=368295\n",
      " DISCARDED=1379045\n",
      " SKIPPED=43840578\n",
      "*** CUBES=5800\n",
      " MATCHED=125888\n",
      " NOT MATCHED=371947\n",
      " DISCARDED=1392859\n",
      " SKIPPED=44623100\n",
      "*** CUBES=5900\n",
      " MATCHED=128940\n",
      " NOT MATCHED=392228\n",
      " DISCARDED=1469584\n",
      " SKIPPED=45326000\n",
      "*** CUBES=6000\n",
      " MATCHED=131456\n",
      " NOT MATCHED=410893\n",
      " DISCARDED=1540436\n",
      " SKIPPED=46036397\n",
      "*** CUBES=6100\n",
      " MATCHED=133591\n",
      " NOT MATCHED=427876\n",
      " DISCARDED=1605176\n",
      " SKIPPED=46754597\n",
      "*** CUBES=6200\n",
      " MATCHED=136371\n",
      " NOT MATCHED=442073\n",
      " DISCARDED=1659541\n",
      " SKIPPED=47485949\n",
      "*** CUBES=6300\n",
      " MATCHED=138638\n",
      " NOT MATCHED=454034\n",
      " DISCARDED=1705697\n",
      " SKIPPED=48227761\n",
      "*** CUBES=6400\n",
      " MATCHED=141844\n",
      " NOT MATCHED=464515\n",
      " DISCARDED=1746418\n",
      " SKIPPED=48976461\n",
      "*** CUBES=6500\n",
      " MATCHED=145234\n",
      " NOT MATCHED=471993\n",
      " DISCARDED=1775305\n",
      " SKIPPED=49739994\n",
      "*** CUBES=6600\n",
      " MATCHED=148409\n",
      " NOT MATCHED=478164\n",
      " DISCARDED=1799165\n",
      " SKIPPED=50509866\n",
      "*** CUBES=6700\n",
      " MATCHED=151664\n",
      " NOT MATCHED=481748\n",
      " DISCARDED=1813082\n",
      " SKIPPED=51292266\n",
      "*** CUBES=6800\n",
      " MATCHED=153254\n",
      " NOT MATCHED=483392\n",
      " DISCARDED=1819486\n",
      " SKIPPED=52084170\n",
      "*** CUBES=6900\n",
      " MATCHED=153254\n",
      " NOT MATCHED=483392\n",
      " DISCARDED=1819486\n",
      " SKIPPED=52884170\n",
      "*** CUBES=7000\n",
      " MATCHED=155288\n",
      " NOT MATCHED=499055\n",
      " DISCARDED=1875355\n",
      " SKIPPED=53612566\n",
      "*** CUBES=7100\n",
      " MATCHED=157744\n",
      " NOT MATCHED=518558\n",
      " DISCARDED=1945050\n",
      " SKIPPED=54323282\n",
      "*** CUBES=7200\n",
      " MATCHED=160065\n",
      " NOT MATCHED=535242\n",
      " DISCARDED=2005277\n",
      " SKIPPED=55046282\n",
      "*** CUBES=7300\n",
      " MATCHED=162426\n",
      " NOT MATCHED=549923\n",
      " DISCARDED=2058401\n",
      " SKIPPED=55778390\n",
      "*** CUBES=7400\n",
      " MATCHED=164912\n",
      " NOT MATCHED=562519\n",
      " DISCARDED=2103821\n",
      " SKIPPED=56520288\n",
      "*** CUBES=7500\n",
      " MATCHED=168049\n",
      " NOT MATCHED=573096\n",
      " DISCARDED=2141543\n",
      " SKIPPED=57271888\n",
      "*** CUBES=7600\n",
      " MATCHED=171072\n",
      " NOT MATCHED=580854\n",
      " DISCARDED=2169302\n",
      " SKIPPED=58036270\n",
      "*** CUBES=7700\n",
      " MATCHED=174149\n",
      " NOT MATCHED=586990\n",
      " DISCARDED=2191309\n",
      " SKIPPED=58808026\n",
      "*** CUBES=7800\n",
      " MATCHED=177115\n",
      " NOT MATCHED=590569\n",
      " DISCARDED=2204232\n",
      " SKIPPED=59591426\n",
      "*** CUBES=7900\n",
      " MATCHED=178161\n",
      " NOT MATCHED=591821\n",
      " DISCARDED=2208758\n",
      " SKIPPED=60385616\n",
      "*** CUBES=8000\n",
      " MATCHED=178161\n",
      " NOT MATCHED=591821\n",
      " DISCARDED=2208758\n",
      " SKIPPED=61185616\n",
      "*** MATCHED=178161\n",
      "*** NOT MATCHED=591821\n",
      "*** DISCARDED=2208758\n",
      "*** SKIPPED=61193616\n",
      "WARNING:tensorflow:From <ipython-input-7-995884d2e25b>:125: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/tb_tear_model2.ckpt\n",
      "#####################################################################\n",
      "INFO:tensorflow:Restoring parameters from /media/1KGB_ILAN/papyrus/models/tb_tear_model1.ckpt\n",
      "Model restored.\n",
      "step 49, training accuracy 0.73\n",
      "step 99, training accuracy 0.72\n",
      "step 149, training accuracy 0.78\n",
      "step 199, training accuracy 0.75\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 249, training accuracy 0.85\n",
      "step 299, training accuracy 0.70\n",
      "step 349, training accuracy 0.87\n",
      "step 399, training accuracy 0.82\n",
      "step 449, training accuracy 0.82\n",
      "step 499, training accuracy 0.87\n",
      "step 549, training accuracy 0.78\n",
      "step 599, training accuracy 0.83\n",
      "step 649, training accuracy 0.90\n",
      "step 699, training accuracy 0.82\n",
      "step 749, training accuracy 0.88\n",
      "step 799, training accuracy 0.73\n",
      "step 849, training accuracy 0.85\n",
      "step 899, training accuracy 0.93\n",
      "step 949, training accuracy 0.78\n",
      "step 999, training accuracy 0.85\n",
      "step 1049, training accuracy 0.85\n",
      "step 1099, training accuracy 0.83\n",
      "step 1149, training accuracy 0.90\n",
      "step 1199, training accuracy 0.82\n",
      "step 1249, training accuracy 0.90\n",
      "step 1299, training accuracy 0.90\n",
      "step 1349, training accuracy 0.87\n",
      "step 1399, training accuracy 0.93\n",
      "step 1449, training accuracy 0.95\n",
      "step 1499, training accuracy 0.85\n",
      "step 1549, training accuracy 0.92\n",
      "step 1599, training accuracy 0.88\n",
      "step 1649, training accuracy 0.92\n",
      "step 1699, training accuracy 0.93\n",
      "step 1749, training accuracy 0.88\n",
      "step 1799, training accuracy 0.92\n",
      "step 1849, training accuracy 0.83\n",
      "step 1899, training accuracy 0.93\n",
      "step 1949, training accuracy 0.93\n",
      "step 1999, training accuracy 0.92\n",
      "Optimization Finished!\n",
      "step 1999, training accuracy 0.916667\n",
      "Model saved in file: /media/1KGB_ILAN/papyrus/models/tb_tear_model2.ckpt\n",
      "#####################################################################\n",
      "TRAINING ENDED\n",
      "#####################################################################\n",
      " \n",
      " \n"
     ]
    }
   ],
   "source": [
    "# RUN2 - take 2nd large pieces and train on it\n",
    "# train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#    parallelize_build_train_set(cubes_set, ROOT_FOLDER + \"train_concats2/\", 1, 7, PARALLEL_build_train_set_for_binary_labeling)\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", \\\n",
    "                                 100, -1, 400, -1, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 12, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep_tb(250)\n",
    "writer = tf.summary.FileWriter(ROOT_FOLDER + \"tf/\" + \"run2\")\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tb_tear_model2.ckpt\", ROOT_FOLDER + \"models/tb_tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-01T23:05:02.443698Z",
     "start_time": "2019-06-01T22:09:12.687507Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", \\\n",
    "#                                  100, -1, 400, -1, max_cols=8, max_rows=5)\n",
    "# tf.reset_default_graph()\n",
    "# model_tf_deep(250)\n",
    "# len_cubes = len(cubes_set)\n",
    "# bat_cubes = 1000\n",
    "# # COPY MANUALLY!!!\n",
    "# #shutil.copy(ROOT_FOLDER + \"models/tear_model1.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")\n",
    "# for start_cubes in range(0, len_cubes, bat_cubes):\n",
    "#     end_cubes = ((start+bat_cubes) if start+bat_cubes < len_cubes else (len_cubes - 1))\n",
    "#     BATCHES = int((end_cubes - start_cubes) / 10) # smaller train batches\n",
    "#     train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#         NEW_build_train_set_for_binary_labeling(cubes_set[start_cubes:end_cubes], cubes_set, CUBE_SIZE, \n",
    "#                                                 ROOT_FOLDER + \"train_concats2/\", 1, 5)\n",
    "#     train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")  \n",
    "# BATCHES = 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# MULTI PROCESS\n",
    "import multiprocessing\n",
    "from itertools import product, repeat\n",
    "\n",
    "num_cores = 4# multiprocessing.cpu_count()\n",
    "num_partitions = num_cores * 4\n",
    "\n",
    "def chunkify(lst,n):\n",
    "    return [lst[i::n] for i in range(n)]\n",
    "\n",
    "def parallelize_build_train_set(cubes_set, folder, enrich_factor, tolerance_factor, func):\n",
    "    adjs_list = chunkify(cubes_set, num_partitions)\n",
    "    print('list:', adjs_list)\n",
    "    # test block for running serialy    \n",
    "    # results = []\n",
    "    # for item in zip(np.arange(num_partitions), df_split):\n",
    "    #    results.append(func(item))\n",
    "    # df = pd.concat(results, ignore_index=True)\n",
    "\n",
    "    # parallel block\n",
    "    pool = multiprocessing.Pool(num_cores)\n",
    "    # import pdb; pdb.set_trace()\n",
    "    args = zip(repeat(cubes_set), adjs_list, repeat(CUBE_SIZE), \\\n",
    "               repeat(ROOT_FOLDER + folder), repeat(enrich_factor), repeat(tolerance_factor))\n",
    "\n",
    "    results = pool.map(func, args)\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    is_enriched = []\n",
    "    \n",
    "    for result in results:\n",
    "        train_imgs.append(result[0])\n",
    "        train_lbls.append(result[1])\n",
    "        train_x_delta.append(result[2])\n",
    "        train_y_delta.append(result[3])\n",
    "        is_enriched.append(result[4])\n",
    "    \n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Parallel RUN2 - take 2nd large pieces and train on it\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", 100, -1, 400, -1, max_cols=9, max_rows=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RE-RUN2 - take 2nd large pieces and train on it\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats2/\")\n",
    "print(len(train_imgs))\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"model_binary/tear_model2.ckpt\", ROOT_FOLDER + \"model_binary/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-10-31T06:26:14.636Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/tb_tear_model3.ckpt\n",
      "#####################################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /media/1KGB_ILAN/papyrus/models/tb_tear_model2.ckpt\n",
      "Model restored.\n",
      "step 49, training accuracy 0.98\n",
      "step 99, training accuracy 1.00\n",
      "step 149, training accuracy 0.97\n",
      "step 199, training accuracy 0.97\n",
      "step 249, training accuracy 1.00\n"
     ]
    }
   ],
   "source": [
    "# OPTIONAL RUN3 - take 3rd large pieces and train on it OR TEST in next block\n",
    "\n",
    "#cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "# train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#     NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\")\n",
    "# tf.reset_default_graph()\n",
    "# model_tf_deep_tb(250)    \n",
    "# writer = tf.summary.FileWriter(ROOT_FOLDER + \"tf/\" + \"run3\")\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tb_tear_model3.ckpt\", ROOT_FOLDER + \"models/tb_tear_model2.ckpt\")\n",
    "# train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:20:39.740469Z",
     "start_time": "2019-06-29T12:43:38.072116Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# TEST3 - take 1 piece and cross- validate on this (uncomment all for full test run)\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/tear_model2.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RE-TEST3 - take 1 piece and cross- validate on this (uncomment all for full test run)\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats3/\")\n",
    "# tf.reset_default_graph()\n",
    "# model_tf_deep(250, 1)    \n",
    "# validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"model_binary/tear_model3.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#### STOP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(train_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(x[1] == 1 for x in train_lbls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "step 49, training accuracy 0.78\n",
    "step 99, training accuracy 0.84\n",
    "step 149, training accuracy 0.9\n",
    "step 199, training accuracy 0.92\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.84\n",
    "step 349, training accuracy 0.92\n",
    "step 399, training accuracy 0.86\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.86\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deper network\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "step 49, training accuracy 0.84\n",
    "step 99, training accuracy 0.94\n",
    "step 149, training accuracy 0.86\n",
    "step 199, training accuracy 0.92\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.96\n",
    "step 399, training accuracy 0.96\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.96\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network on GCP\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/home/il239838/files/model_binary/tear_model1.ckpt\n",
    "#####################################################################\n",
    "WARNING:tensorflow:From /home/il239838/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
    "Instructions for updating:\n",
    "Use `tf.global_variables_initializer` instead.\n",
    "step 49, training accuracy 0.82\n",
    "step 99, training accuracy 0.92\n",
    "step 149, training accuracy 0.72\n",
    "step 199, training accuracy 0.8\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.88\n",
    "step 349, training accuracy 0.94\n",
    "step 399, training accuracy 0.84\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.84\n",
    "Model saved in file: /home/il239838/files/model_binary/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "  \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.76\n",
    "step 99, training accuracy 0.82\n",
    "step 149, training accuracy 0.96\n",
    "step 199, training accuracy 0.86\n",
    "step 249, training accuracy 0.76\n",
    "step 299, training accuracy 0.82\n",
    "step 349, training accuracy 0.86\n",
    "step 399, training accuracy 0.88\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.88\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.88\n",
    "step 99, training accuracy 0.88\n",
    "step 149, training accuracy 0.88\n",
    "step 199, training accuracy 0.88\n",
    "step 249, training accuracy 0.92\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.86\n",
    "step 399, training accuracy 0.94\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.94\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network on GCP\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/home/il239838/files/model_binary/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /home/il239838/files/model_binary/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.88\n",
    "step 99, training accuracy 0.96\n",
    "step 149, training accuracy 0.84\n",
    "step 199, training accuracy 0.84\n",
    "step 249, training accuracy 0.84\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.98\n",
    "step 399, training accuracy 0.9\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.9\n",
    "Model saved in file: /home/il239838/files/model_binary/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    ">>> step 100\n",
    "step 0-99, precision 0.695652, recall 0.640000, f_score 0.666667\n",
    ">>> step 200\n",
    "step 100-199, precision 0.512500, recall 0.732143, f_score 0.602941\n",
    ">>> step 300\n",
    "step 200-299, precision 0.350427, recall 0.732143, f_score 0.473988\n",
    ">>> step 400\n",
    "step 300-399, precision 0.408805, recall 0.677083, f_score 0.509804\n",
    ">>> step 500\n",
    "step 400-499, precision 0.366834, recall 0.651786, f_score 0.469453\n",
    ">>> step 600\n",
    "step 500-599, precision 0.349282, recall 0.651786, f_score 0.454829\n",
    ">>> step 700\n",
    "step 600-699, precision 0.376471, recall 0.662069, f_score 0.480000\n",
    ">>> step 800\n",
    "step 700-799, precision 0.383142, recall 0.595238, f_score 0.466200\n",
    ">>> step 900\n",
    "step 800-899, precision 0.394649, recall 0.617801, f_score 0.481633\n",
    ">>> step 1000\n",
    "step 900-999, precision 0.407821, recall 0.651786, f_score 0.501718\n",
    ">>> step 1100\n",
    "step 1000-1099, precision 0.404432, recall 0.651786, f_score 0.499145\n",
    ">>> step 1200\n",
    "step 1100-1199, precision 0.423559, recall 0.657588, f_score 0.515244\n",
    ">>> step 1300\n",
    "step 1200-1299, precision 0.408983, recall 0.662835, f_score 0.505848\n",
    ">>> step 1400\n",
    "step 1300-1399, precision 0.405640, recall 0.667857, f_score 0.504723\n",
    ">>> step 1500\n",
    "step 1400-1499, precision 0.415020, recall 0.670927, f_score 0.512821\n",
    ">>> step 1600\n",
    "step 1500-1599, precision 0.402852, recall 0.672619, f_score 0.503902\n",
    ">>> step 1700\n",
    "step 1600-1699, precision 0.401681, recall 0.647696, f_score 0.495851\n",
    ">>> step 1800\n",
    "step 1700-1799, precision 0.387987, recall 0.647696, f_score 0.485279\n",
    ">>> step 1900\n",
    "step 1800-1899, precision 0.385484, recall 0.647696, f_score 0.483316\n",
    ">>> step 2000\n",
    "step 1900-1999, precision 0.388802, recall 0.647668, f_score 0.485909\n",
    ">>> step 2100\n",
    "step 2000-2099, precision 0.379161, recall 0.650124, f_score 0.478976\n",
    ">>> step 2200\n",
    "step 2100-2199, precision 0.374286, recall 0.650124, f_score 0.475068\n",
    ">>> step 2300\n",
    "step 2200-2299, precision 0.381868, recall 0.651054, f_score 0.481385\n",
    ">>> step 2400\n",
    "step 2300-2399, precision 0.392622, recall 0.659292, f_score 0.492155\n",
    ">>> step 2500\n",
    "step 2400-2499, precision 0.393604, recall 0.665281, f_score 0.494590\n",
    ">>> step 2600\n",
    "step 2500-2599, precision 0.392601, recall 0.664646, f_score 0.493623\n",
    ">>> step 2700\n",
    "step 2600-2699, precision 0.389810, recall 0.664646, f_score 0.491412\n",
    ">>> step 2800\n",
    "step 2700-2799, precision 0.384884, recall 0.655446, f_score 0.484982\n",
    ">>> step 2900\n",
    "step 2800-2899, precision 0.390572, recall 0.664122, f_score 0.491873\n",
    ">>> step 3000\n",
    "step 2900-2999, precision 0.401064, recall 0.679279, f_score 0.504348\n",
    ">>> step 3100\n",
    "step 3000-3099, precision 0.418534, recall 0.695431, f_score 0.522568\n",
    ">>> step 3200\n",
    "step 3100-3199, precision 0.430129, recall 0.704545, f_score 0.534154\n",
    ">>> step 3300\n",
    "step 3200-3299, precision 0.446036, recall 0.716258, f_score 0.549735\n",
    ">>> step 3400\n",
    "step 3300-3399, precision 0.448563, recall 0.722388, f_score 0.553459\n",
    ">>> step 3500\n",
    "step 3400-3499, precision 0.455204, recall 0.719599, f_score 0.557650\n",
    ">>> step 3600\n",
    "step 3500-3599, precision 0.456560, recall 0.718271, f_score 0.558266\n",
    ">>> step 3700\n",
    "step 3600-3699, precision 0.457841, recall 0.724000, f_score 0.560950\n",
    ">>> step 3800\n",
    "step 3700-3799, precision 0.455285, recall 0.720721, f_score 0.558047\n",
    ">>> step 3900\n",
    "step 3800-3899, precision 0.459969, recall 0.724351, f_score 0.562650\n",
    ">>> step 4000\n",
    "step 3900-3999, precision 0.445289, recall 0.724351, f_score 0.551529\n",
    ">>> step 4100\n",
    "step 4000-4099, precision 0.442598, recall 0.724351, f_score 0.549461\n",
    ">>> step 4200\n",
    "step 4100-4199, precision 0.445596, recall 0.725301, f_score 0.552040\n",
    ">>> step 4300\n",
    "step 4200-4299, precision 0.429793, recall 0.724760, f_score 0.539597\n",
    ">>> step 4400\n",
    "step 4300-4399, precision 0.426450, recall 0.724760, f_score 0.536955\n",
    ">>> step 4500\n",
    "step 4400-4499, precision 0.427491, recall 0.727485, f_score 0.538528\n",
    ">>> step 4600\n",
    "step 4500-4599, precision 0.427310, recall 0.723820, f_score 0.537377\n",
    ">>> step 4700\n",
    "step 4600-4699, precision 0.422999, recall 0.723820, f_score 0.533956\n",
    ">>> step 4800\n",
    "step 4700-4799, precision 0.419893, recall 0.723820, f_score 0.531474\n",
    ">>> step 4900\n",
    "step 4800-4899, precision 0.423740, recall 0.723669, f_score 0.534504\n",
    ">>> step 5000\n",
    "step 4900-4999, precision 0.428387, recall 0.725683, f_score 0.538742\n",
    ">>> step 5100\n",
    "step 5000-5099, precision 0.417348, recall 0.725683, f_score 0.529928\n",
    ">>> step 5200\n",
    "step 5100-5199, precision 0.414482, recall 0.725683, f_score 0.527612\n",
    ">>> step 5300\n",
    "step 5200-5299, precision 0.412935, recall 0.725683, f_score 0.526358\n",
    ">>> step 5400\n",
    "step 5300-5399, precision 0.418093, recall 0.720759, f_score 0.529207\n",
    ">>> step 5500\n",
    "step 5400-5499, precision 0.409934, recall 0.720294, f_score 0.522502\n",
    ">>> step 5600\n",
    "step 5500-5599, precision 0.409280, recall 0.719665, f_score 0.521805\n",
    ">>> step 5700\n",
    "step 5600-5699, precision 0.411176, recall 0.716189, f_score 0.522422\n",
    ">>> step 5800\n",
    "step 5700-5799, precision 0.413056, recall 0.707921, f_score 0.521707\n",
    ">>> step 5900\n",
    "step 5800-5899, precision 0.408427, recall 0.701061, f_score 0.516152\n",
    ">>> step 6000\n",
    "step 5900-5999, precision 0.404113, recall 0.701061, f_score 0.512694\n",
    ">>> step 6100\n",
    "step 6000-6099, precision 0.399565, recall 0.700382, f_score 0.508839\n",
    ">>> step 6200\n",
    "step 6100-6199, precision 0.398913, recall 0.700382, f_score 0.508310\n",
    ">>> step 6300\n",
    "step 6200-6299, precision 0.402681, recall 0.704503, f_score 0.512453\n",
    ">>> step 6400\n",
    "step 6300-6399, precision 0.404178, recall 0.709441, f_score 0.514970\n",
    ">>> step 6500\n",
    "step 6400-6499, precision 0.402955, recall 0.713255, f_score 0.514974\n",
    ">>> step 6600\n",
    "step 6500-6599, precision 0.402532, recall 0.713645, f_score 0.514730\n",
    ">>> step 6700\n",
    "step 6600-6699, precision 0.413011, recall 0.712585, f_score 0.522933\n",
    ">>> step 6800\n",
    "step 6700-6799, precision 0.416506, recall 0.713813, f_score 0.526059\n",
    ">>> step 6900\n",
    "step 6800-6899, precision 0.421002, recall 0.712439, f_score 0.529253\n",
    ">>> step 7000\n",
    "step 6900-6999, precision 0.428773, recall 0.709176, f_score 0.534427\n",
    ">>> step 7100\n",
    "step 7000-7099, precision 0.430497, recall 0.704718, f_score 0.534488\n",
    ">>> step 7200\n",
    "step 7100-7199, precision 0.437929, recall 0.695273, f_score 0.537381\n",
    ">>> step 7300\n",
    "step 7200-7299, precision 0.440254, recall 0.693133, f_score 0.538483\n",
    ">>> step 7400\n",
    "step 7300-7399, precision 0.441348, recall 0.690577, f_score 0.538525\n",
    ">>> step 7500\n",
    "step 7400-7499, precision 0.439821, recall 0.688375, f_score 0.536719\n",
    ">>> step 7600\n",
    "step 7500-7599, precision 0.440693, recall 0.683196, f_score 0.535782\n",
    ">>> step 7700\n",
    "step 7600-7699, precision 0.440618, recall 0.676152, f_score 0.533547\n",
    ">>> step 7800\n",
    "step 7700-7799, precision 0.435618, recall 0.676152, f_score 0.529865\n",
    ">>> step 7900\n",
    "step 7800-7899, precision 0.431701, recall 0.674044, f_score 0.526316\n",
    ">>> step 8000\n",
    "step 7900-7999, precision 0.429482, recall 0.668651, f_score 0.523021\n",
    ">>> step 8100\n",
    "step 8000-8099, precision 0.428027, recall 0.667768, f_score 0.521672\n",
    ">>> step 8200\n",
    "step 8100-8199, precision 0.429717, recall 0.664057, f_score 0.521784\n",
    ">>> step 8300\n",
    "step 8200-8299, precision 0.428451, recall 0.664057, f_score 0.520849\n",
    ">>> step 8400\n",
    "step 8300-8399, precision 0.429589, recall 0.660438, f_score 0.520569\n",
    ">>> step 8500\n",
    "step 8400-8499, precision 0.428870, recall 0.660438, f_score 0.520041\n",
    ">>> step 8600\n",
    "step 8500-8599, precision 0.427618, recall 0.660438, f_score 0.519119\n",
    ">>> step 8700\n",
    "step 8600-8699, precision 0.424528, recall 0.655478, f_score 0.515310\n",
    ">>> step 8800\n",
    "step 8700-8799, precision 0.421074, recall 0.655478, f_score 0.512757\n",
    ">>> step 8900\n",
    "step 8800-8899, precision 0.420032, recall 0.650814, f_score 0.510555\n",
    ">>> step 9000\n",
    "step 8900-8999, precision 0.415667, recall 0.650814, f_score 0.507317\n",
    ">>> step 9100\n",
    "step 9000-9099, precision 0.414201, recall 0.646154, f_score 0.504808\n",
    ">>> step 9200\n",
    "step 9100-9199, precision 0.414510, recall 0.642944, f_score 0.504053\n",
    ">>> step 9300\n",
    "step 9200-9299, precision 0.413793, recall 0.639138, f_score 0.502352\n",
    ">>> step 9400\n",
    "step 9300-9399, precision 0.408101, recall 0.639138, f_score 0.498134\n",
    ">>> step 9500\n",
    "step 9400-9499, precision 0.407789, recall 0.639138, f_score 0.497902\n",
    ">>> step 9600\n",
    "step 9500-9599, precision 0.408248, recall 0.639218, f_score 0.498268\n",
    ">>> step 9700\n",
    "step 9600-9699, precision 0.406097, recall 0.639218, f_score 0.496663\n",
    ">>> step 9800\n",
    "step 9700-9799, precision 0.408482, recall 0.639487, f_score 0.498524\n",
    ">>> step 9900\n",
    "step 9800-9899, precision 0.407298, recall 0.639098, f_score 0.497524\n",
    ">>> step 10000\n",
    "step 9900-9999, precision 0.406250, recall 0.639098, f_score 0.496741\n",
    ">>> step 10100\n",
    "step 10000-10099, precision 0.407407, recall 0.636312, f_score 0.496758\n",
    ">>> step 10200\n",
    "step 10100-10199, precision 0.408892, recall 0.636415, f_score 0.497892\n",
    ">>> step 10300\n",
    "step 10200-10299, precision 0.407852, recall 0.636415, f_score 0.497120\n",
    ">>> step 10400\n",
    "step 10300-10399, precision 0.408796, recall 0.637079, f_score 0.498024\n",
    ">>> step 10500\n",
    "step 10400-10499, precision 0.406452, recall 0.637079, f_score 0.496280\n",
    ">>> step 10600\n",
    "step 10500-10599, precision 0.405540, recall 0.637989, f_score 0.495875\n",
    ">>> step 10700\n",
    "step 10600-10699, precision 0.403583, recall 0.636918, f_score 0.494087\n",
    ">>> step 10800\n",
    "step 10700-10799, precision 0.401748, recall 0.636918, f_score 0.492710\n",
    ">>> step 10900\n",
    "step 10800-10899, precision 0.402490, recall 0.636066, f_score 0.493011\n",
    ">>> step 11000\n",
    "step 10900-10999, precision 0.400619, recall 0.633496, f_score 0.490836\n",
    ">>> step 11100\n",
    "step 11000-11099, precision 0.396798, recall 0.633496, f_score 0.487958\n",
    ">>> step 11200\n",
    "step 11100-11199, precision 0.396944, recall 0.629510, f_score 0.486880\n",
    ">>> step 11300\n",
    "step 11200-11299, precision 0.397928, recall 0.628496, f_score 0.487316\n",
    ">>> step 11400\n",
    "step 11300-11399, precision 0.398474, recall 0.625521, f_score 0.486826\n",
    ">>> step 11500\n",
    "step 11400-11499, precision 0.397233, recall 0.620690, f_score 0.484435\n",
    ">>> step 11600\n",
    "step 11500-11599, precision 0.396189, recall 0.620690, f_score 0.483658\n",
    ">>> step 11700\n",
    "step 11600-11699, precision 0.397906, recall 0.620725, f_score 0.484945\n",
    ">>> step 11800\n",
    "step 11700-11799, precision 0.397856, recall 0.618999, f_score 0.484381\n",
    ">>> step 11900\n",
    "step 11800-11899, precision 0.396184, recall 0.618999, f_score 0.483139\n",
    ">>> step 12000\n",
    "step 11900-11999, precision 0.394779, recall 0.618999, f_score 0.482094\n",
    ">>> step 12100\n",
    "step 12000-12099, precision 0.397894, recall 0.619473, f_score 0.484554\n",
    ">>> step 12200\n",
    "step 12100-12199, precision 0.402704, recall 0.621242, f_score 0.488652\n",
    ">>> step 12300\n",
    "step 12200-12299, precision 0.408271, recall 0.618464, f_score 0.491852\n",
    ">>> step 12400\n",
    "step 12300-12399, precision 0.411149, recall 0.615207, f_score 0.492893\n",
    ">>> step 12500\n",
    "step 12400-12499, precision 0.414962, recall 0.612991, f_score 0.494902\n",
    ">>> step 12600\n",
    "step 12500-12599, precision 0.416262, recall 0.606275, f_score 0.493614\n",
    ">>> step 12700\n",
    "step 12600-12699, precision 0.419520, recall 0.603456, f_score 0.494951\n",
    ">>> step 12800\n",
    "step 12700-12799, precision 0.418013, recall 0.603456, f_score 0.493901\n",
    ">>> step 12900\n",
    "step 12800-12899, precision 0.417014, recall 0.600943, f_score 0.492362\n",
    ">>> step 13000\n",
    "step 12900-12999, precision 0.412596, recall 0.600943, f_score 0.489269\n",
    ">>> step 13100\n",
    "step 13000-13099, precision 0.410480, recall 0.599745, f_score 0.487383\n",
    ">>> step 13200\n",
    "step 13100-13199, precision 0.408577, recall 0.599745, f_score 0.486039\n",
    ">>> step 13300\n",
    "step 13200-13299, precision 0.407397, recall 0.599745, f_score 0.485203\n",
    ">>> step 13400\n",
    "step 13300-13399, precision 0.406178, recall 0.599409, f_score 0.484228\n",
    ">>> step 13500\n",
    "step 13400-13499, precision 0.401925, recall 0.599409, f_score 0.481193\n",
    ">>> step 13600\n",
    "step 13500-13599, precision 0.401584, recall 0.599409, f_score 0.480948\n",
    ">>> step 13700\n",
    "step 13600-13699, precision 0.400391, recall 0.600335, f_score 0.480389\n",
    ">>> step 13800\n",
    "step 13700-13799, precision 0.399224, recall 0.598753, f_score 0.479042\n",
    ">>> step 13900\n",
    "step 13800-13899, precision 0.395553, recall 0.598919, f_score 0.476442\n",
    ">>> step 14000\n",
    "step 13900-13999, precision 0.393915, recall 0.598432, f_score 0.475098\n",
    ">>> step 14100\n",
    "step 14000-14099, precision 0.393915, recall 0.598432, f_score 0.475098\n",
    ">>> step 14200\n",
    "step 14100-14199, precision 0.392741, recall 0.598432, f_score 0.474244\n",
    ">>> step 14300\n",
    "step 14200-14299, precision 0.393562, recall 0.597536, f_score 0.474560\n",
    ">>> step 14400\n",
    "step 14300-14399, precision 0.390675, recall 0.594374, f_score 0.471463\n",
    ">>> step 14500\n",
    "step 14400-14499, precision 0.389944, recall 0.592924, f_score 0.470474\n",
    ">>> step 14600\n",
    "step 14500-14599, precision 0.386943, recall 0.592924, f_score 0.468283\n",
    ">>> step 14700\n",
    "step 14600-14699, precision 0.386532, recall 0.592924, f_score 0.467983\n",
    ">>> step 14800\n",
    "step 14700-14799, precision 0.387326, recall 0.595152, f_score 0.469258\n",
    ">>> step 14900\n",
    "step 14800-14899, precision 0.386903, recall 0.595343, f_score 0.469007\n",
    ">>> step 15000\n",
    "step 14900-14999, precision 0.388658, recall 0.595694, f_score 0.470403\n",
    ">>> step 15100\n",
    "step 15000-15099, precision 0.387264, recall 0.595304, f_score 0.469260\n",
    ">>> step 15200\n",
    "step 15100-15199, precision 0.386923, recall 0.596679, f_score 0.469435\n",
    ">>> step 15300\n",
    "step 15200-15299, precision 0.387583, recall 0.597950, f_score 0.470315\n",
    ">>> step 15400\n",
    "step 15300-15399, precision 0.384228, recall 0.599214, f_score 0.468222\n",
    ">>> step 15500\n",
    "step 15400-15499, precision 0.383070, recall 0.599214, f_score 0.467361\n",
    ">>> step 15600\n",
    "step 15500-15599, precision 0.384500, recall 0.599143, f_score 0.468403\n",
    ">>> step 15700\n",
    "step 15600-15699, precision 0.386607, recall 0.601239, f_score 0.470606\n",
    ">>> step 15800\n",
    "step 15700-15799, precision 0.382607, recall 0.601239, f_score 0.467630\n",
    ">>> step 15900\n",
    "step 15800-15899, precision 0.382626, recall 0.603309, f_score 0.468269\n",
    ">>> step 16000\n",
    "step 15900-15999, precision 0.383248, recall 0.603296, f_score 0.468731\n",
    ">>> step 16100\n",
    "step 16000-16099, precision 0.382317, recall 0.603296, f_score 0.468034\n",
    ">>> step 16200\n",
    "step 16100-16199, precision 0.381946, recall 0.603296, f_score 0.467756\n",
    ">>> step 16300\n",
    "step 16200-16299, precision 0.379460, recall 0.603296, f_score 0.465887\n",
    ">>> step 16400\n",
    "step 16300-16399, precision 0.382472, recall 0.604669, f_score 0.468563\n",
    ">>> step 16500\n",
    "step 16400-16499, precision 0.385124, recall 0.605644, f_score 0.470843\n",
    ">>> step 16600\n",
    "step 16500-16599, precision 0.386226, recall 0.609174, f_score 0.472732\n",
    ">>> step 16700\n",
    "step 16600-16699, precision 0.385509, recall 0.609174, f_score 0.472195\n",
    ">>> step 16800\n",
    "step 16700-16799, precision 0.387558, recall 0.609641, f_score 0.473870\n",
    ">>> step 16900\n",
    "step 16800-16899, precision 0.389587, recall 0.611689, f_score 0.476004\n",
    ">>> step 17000\n",
    "step 16900-16999, precision 0.392534, recall 0.612858, f_score 0.478555\n",
    ">>> step 17100\n",
    "step 17000-17099, precision 0.393763, recall 0.611711, f_score 0.479115\n",
    ">>> step 17200\n",
    "step 17100-17199, precision 0.392442, recall 0.611711, f_score 0.478136\n",
    ">>> step 17300\n",
    "step 17200-17299, precision 0.394895, recall 0.614296, f_score 0.480746\n",
    ">>> step 17400\n",
    "step 17300-17399, precision 0.397101, recall 0.616644, f_score 0.483100\n",
    ">>> step 17500\n",
    "step 17400-17499, precision 0.401132, recall 0.617085, f_score 0.486208\n",
    ">>> step 17600\n",
    "step 17500-17599, precision 0.399220, recall 0.617085, f_score 0.484801\n",
    ">>> step 17700\n",
    "step 17600-17699, precision 0.400259, recall 0.617539, f_score 0.485707\n",
    ">>> step 17800\n",
    "step 17700-17799, precision 0.402402, recall 0.618734, f_score 0.487653\n",
    ">>> step 17900\n",
    "step 17800-17899, precision 0.403036, recall 0.617830, f_score 0.487836\n",
    ">>> step 18000\n",
    "step 17900-17999, precision 0.404838, recall 0.618076, f_score 0.489231\n",
    ">>> step 18100\n",
    "step 18000-18099, precision 0.407634, recall 0.618758, f_score 0.491482\n",
    ">>> step 18200\n",
    "step 18100-18199, precision 0.406473, recall 0.618682, f_score 0.490614\n",
    ">>> step 18300\n",
    "step 18200-18299, precision 0.407004, recall 0.619139, f_score 0.491144\n",
    ">>> step 18400\n",
    "step 18300-18399, precision 0.406982, recall 0.619472, f_score 0.491232\n",
    ">>> step 18500\n",
    "step 18400-18499, precision 0.405737, recall 0.619093, f_score 0.490206\n",
    ">>> step 18600\n",
    "step 18500-18599, precision 0.404076, recall 0.620025, f_score 0.489282\n",
    ">>> step 18700\n",
    "step 18600-18699, precision 0.401760, recall 0.620025, f_score 0.487581\n",
    ">>> step 18800\n",
    "step 18700-18799, precision 0.399308, recall 0.620025, f_score 0.485771\n",
    ">>> step 18900\n",
    "step 18800-18899, precision 0.398062, recall 0.620321, f_score 0.484938\n",
    ">>> step 19000\n",
    "step 18900-18999, precision 0.396462, recall 0.620321, f_score 0.483748\n",
    ">>> step 19100\n",
    "step 19000-19099, precision 0.396994, recall 0.620614, f_score 0.484234\n",
    ">>> step 19200\n",
    "step 19100-19199, precision 0.395224, recall 0.620431, f_score 0.482859\n",
    ">>> step 19300\n",
    "step 19200-19299, precision 0.393585, recall 0.620281, f_score 0.481589\n",
    ">>> step 19400\n",
    "step 19300-19399, precision 0.393508, recall 0.620281, f_score 0.481531\n",
    ">>> step 19500\n",
    "step 19400-19499, precision 0.393243, recall 0.621160, f_score 0.481597\n",
    ">>> step 19600\n",
    "step 19500-19599, precision 0.392549, recall 0.621160, f_score 0.481077\n",
    ">>> step 19700\n",
    "step 19600-19699, precision 0.392088, recall 0.621160, f_score 0.480730\n",
    ">>> step 19800\n",
    "step 19700-19799, precision 0.391704, recall 0.621160, f_score 0.480442\n",
    ">>> step 19900\n",
    "step 19800-19899, precision 0.391016, recall 0.621160, f_score 0.479923\n",
    ">>> step 20000\n",
    "step 19900-19999, precision 0.390358, recall 0.620519, f_score 0.479236\n",
    ">>> step 20100\n",
    "step 20000-20099, precision 0.390504, recall 0.620764, f_score 0.479419\n",
    ">>> step 20200\n",
    "step 20100-20199, precision 0.386887, recall 0.621114, f_score 0.476787\n",
    ">>> step 20300\n",
    "step 20200-20299, precision 0.384308, recall 0.621114, f_score 0.474824\n",
    ">>> step 20400\n",
    "step 20300-20399, precision 0.384161, recall 0.621114, f_score 0.474712\n",
    ">>> step 20500\n",
    "step 20400-20499, precision 0.383577, recall 0.621114, f_score 0.474266\n",
    ">>> step 20600\n",
    "step 20500-20599, precision 0.383660, recall 0.622861, f_score 0.474837\n",
    ">>> step 20700\n",
    "step 20600-20699, precision 0.383569, recall 0.624886, f_score 0.475355\n",
    ">>> step 20800\n",
    "step 20700-20799, precision 0.382785, recall 0.624886, f_score 0.474752\n",
    ">>> step 20900\n",
    "step 20800-20899, precision 0.384644, recall 0.627704, f_score 0.476995\n",
    ">>> step 21000\n",
    "step 20900-20999, precision 0.383092, recall 0.627704, f_score 0.475800\n",
    ">>> step 21100\n",
    "step 21000-21099, precision 0.383464, recall 0.626977, f_score 0.475878\n",
    ">>> step 21200\n",
    "step 21100-21199, precision 0.383045, recall 0.626977, f_score 0.475555\n",
    ">>> step 21300\n",
    "step 21200-21299, precision 0.383430, recall 0.626852, f_score 0.475816\n",
    ">>> step 21400\n",
    "step 21300-21399, precision 0.384685, recall 0.628119, f_score 0.477146\n",
    ">>> step 21500\n",
    "step 21400-21499, precision 0.383581, recall 0.628119, f_score 0.476296\n",
    ">>> step 21600\n",
    "step 21500-21599, precision 0.384506, recall 0.629446, f_score 0.477391\n",
    ">>> step 21700\n",
    "step 21600-21699, precision 0.382462, recall 0.629446, f_score 0.475813\n",
    ">>> step 21800\n",
    "step 21700-21799, precision 0.382716, recall 0.628439, f_score 0.475721\n",
    ">>> step 21900\n",
    "step 21800-21899, precision 0.380835, recall 0.628439, f_score 0.474265\n",
    ">>> step 22000\n",
    "step 21900-21999, precision 0.380235, recall 0.628439, f_score 0.473799\n",
    ">>> step 22100\n",
    "step 22000-22099, precision 0.380420, recall 0.629084, f_score 0.474126\n",
    ">>> step 22200\n",
    "step 22100-22199, precision 0.378838, recall 0.629758, f_score 0.473086\n",
    ">>> step 22300\n",
    "step 22200-22299, precision 0.378196, recall 0.628481, f_score 0.472225\n",
    ">>> step 22400\n",
    "step 22300-22399, precision 0.377388, recall 0.627647, f_score 0.471359\n",
    ">>> step 22500\n",
    "step 22400-22499, precision 0.377378, recall 0.626815, f_score 0.471117\n",
    ">>> step 22600\n",
    "step 22500-22599, precision 0.377937, recall 0.628183, f_score 0.471939\n",
    ">>> step 22700\n",
    "step 22600-22699, precision 0.376335, recall 0.628183, f_score 0.470688\n",
    ">>> step 22800\n",
    "step 22700-22799, precision 0.376016, recall 0.628183, f_score 0.470439\n",
    ">>> step 22900\n",
    "step 22800-22899, precision 0.375968, recall 0.629473, f_score 0.470762\n",
    ">>> step 23000\n",
    "step 22900-22999, precision 0.375210, recall 0.629473, f_score 0.470167\n",
    ">>> step 23100\n",
    "step 23000-23099, precision 0.374708, recall 0.630191, f_score 0.469973\n",
    ">>> step 23200\n",
    "step 23100-23199, precision 0.376144, recall 0.631564, f_score 0.471484\n",
    ">>> step 23300\n",
    "step 23200-23299, precision 0.376630, recall 0.632834, f_score 0.472219\n",
    ">>> step 23400\n",
    "step 23300-23399, precision 0.375021, recall 0.632834, f_score 0.470952\n",
    ">>> step 23500\n",
    "step 23400-23499, precision 0.374286, recall 0.633527, f_score 0.470564\n",
    ">>> step 23600\n",
    "step 23500-23599, precision 0.374068, recall 0.634763, f_score 0.470732\n",
    ">>> step 23700\n",
    "step 23600-23699, precision 0.373814, recall 0.635494, f_score 0.470731\n",
    ">>> step 23800\n",
    "step 23700-23799, precision 0.372496, recall 0.635494, f_score 0.469685\n",
    ">>> step 23900\n",
    "step 23800-23899, precision 0.372906, recall 0.635918, f_score 0.470127\n",
    ">>> step 24000\n",
    "step 23900-23999, precision 0.372976, recall 0.634865, f_score 0.469894\n",
    ">>> step 24100\n",
    "step 24000-24099, precision 0.373713, recall 0.634750, f_score 0.470448\n",
    ">>> step 24200\n",
    "step 24100-24199, precision 0.373123, recall 0.634750, f_score 0.469979\n",
    ">>> step 24300\n",
    "step 24200-24299, precision 0.374528, recall 0.635369, f_score 0.471263\n",
    ">>> step 24400\n",
    "step 24300-24399, precision 0.374765, recall 0.635397, f_score 0.471458"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"test %.2f\"%0.12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "for i in range(len(cubes_set)):\n",
    "    if (cubes_set[i][\"file\"]) == \"PX303-Fg001-V-C01-R01_TEAR_8X5_PIECE_2X4\":\n",
    "        plt.imshow(cubes_set[i][\"cube\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "for i in range(len(cubes_set)):\n",
    "    if (cubes_set[i][\"file\"]) == \"PX303-Fg001-V-C01-R01_TEAR_8X5_PIECE_4X4\":\n",
    "        plt.imshow(cubes_set[i][\"cube\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# When trained on a ratio of 1:2 non-match:match - and tested on the same the results are excellend\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=8183\n",
    "*** DISCARDED=1979027\n",
    "TOTAL 27300, precision 0.815258, recall 0.979806, f_score 0.889990\n",
    "TOTAL true = 19065\n",
    "TOTAL error rate = 0.169158\n",
    "\n",
    "# When trained on a ratio of 1:2 non-match:match - and tested on 6:1 (much more non-match) the results are poor\n",
    "# coverage is still excellent but precision is bad - means a lot of false-positive\n",
    "# this usually implies that we'll get a lot of trash in the validation set\n",
    "# hence next attempt will be to train on 6:1 and see it the test is better\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=125210\n",
    "*** DISCARDED=1862000\n",
    "precision 0.237849, recall 0.973684, f_score 0.382309\n",
    "\n",
    "# When trained on a ratio of 9:1 non-match:match - and tested on 6:1 the results are poor\n",
    "# coverage is around 70% and precision is 35 - means a lot of false-positive\n",
    "# this usually implies that we'll get a lot of trash in the validation set\n",
    "# hence next attempt will be to train on 6:1 and see it the test is better\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=125210\n",
    "*** DISCARDED=1862000\n",
    "precision 0.355357, recall 0.737720, f_score 0.479663\n",
    "\n",
    "\n",
    "# Test from 0ct-07\n",
    "=== total 144000 match 62339\n",
    "TOTAL 144000, precision 0.286081, recall 0.952518, f_score 0.440009\n",
    "TOTAL true = 18723\n",
    "TOTAL error rate = 0.315236"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#debug\n",
    "\n",
    "short_name = \"PX303-Fg006-V-C02-R02\"\n",
    "# image = read_and_crop(img_name, x_start, x_end, y_start, y_end)\n",
    "\n",
    "cubes_set = []\n",
    "col_cut = 3\n",
    "row_cut = 2\n",
    "pieces = []\n",
    "piece = {}\n",
    "image = img.imread(ROOT_FOLDER + \"test_fragments/PX303Fg006_3X2_1X1.jpg\")\n",
    "image = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "piece[\"cut\"] = image[:,:,0]\n",
    "piece[\"col\"] = 1\n",
    "piece[\"row\"] = 1\n",
    "piece[\"col_px\"] = 0\n",
    "piece[\"row_px\"] = 0\n",
    "pieces.append(piece)\n",
    "piece = {}\n",
    "image = img.imread(ROOT_FOLDER + \"test_fragments/PX303Fg006_3X2_2X1.jpg\")\n",
    "image = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "piece[\"cut\"] = image[:,:,0]\n",
    "piece[\"col\"] = 2\n",
    "piece[\"row\"] = 1\n",
    "piece[\"col_px\"] = 2300\n",
    "piece[\"row_px\"] = 0\n",
    "pieces.append(piece)\n",
    "\n",
    "for piece in pieces:\n",
    "    # print(\"PRE_PROCESS:::\"+\"PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"]))\n",
    "    fragment_name = short_name + \"_TEAR_\"+str(col_cut)+\"X\"+str(row_cut)+\"_PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "    fragment_file_name = short_name + \"_\"+str(col_cut)+\"X\"+str(row_cut)+\"_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "    cubes = VAL_slice_TEAR_to_static_slices(fragment_name, piece)\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for cube in cubes:\n",
    "        cube[\"tear\"] = str(col_cut)+\"X\"+str(row_cut)\n",
    "        cube[\"piece_col\"] = piece[\"col\"]\n",
    "        cube[\"piece_row\"] = piece[\"row\"]\n",
    "    # print(\"File: %s >>> cubes: %d\"%(file_, len(cubes)))\n",
    "    cubes_set.extend(cubes)\n",
    "    \n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 7)\n",
    "import pdb; pdb.set_trace()    \n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"model_binary/tear_model3.ckpt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:32:29.152185Z",
     "start_time": "2019-04-03T09:38:59.768856Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#TRAIN 3\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T13:33:46.532102Z",
     "start_time": "2019-04-03T12:32:32.371972Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#TRAIN 2\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", \\\n",
    "                                 100, -1, 400, -1, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T13:49:29.331063Z",
     "start_time": "2019-04-03T13:34:34.742578Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# validate 1\n",
    "cubes_set = pre_process_training(\"PX303-Fg001-V-C01-R01-D05032015-T112520-ML638__006.jpg\", max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "  NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 5)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/tear_model2.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############2_3_val_1\n",
    "=== total 91300 match 12738 /// 1,5 => 1,5\n",
    "TOTAL 91300, precision 0.475899, recall 0.349899, f_score 0.403286\n",
    "TOTAL true = 17325\n",
    "TOTAL error rate = 0.196484\n",
    "=== total 91900 match 12457 /// 1,7 => 1,5\n",
    "TOTAL 91900, precision 0.511760, recall 0.364765, f_score 0.425937\n",
    "TOTAL true = 17477\n",
    "TOTAL error rate = 0.186986\n",
    "=== total 91000 match 20063 /// 2,7 => 1,5\n",
    "TOTAL 91000, precision 0.516672, recall 0.600266, f_score 0.555341\n",
    "TOTAL true = 17269\n",
    "TOTAL error rate = 0.182418\n",
    "=== total 91100 match 19002 /// 3,7 => 1,5\n",
    "TOTAL 91100, precision 0.501474, recall 0.552022, f_score 0.525535\n",
    "TOTAL true = 17262\n",
    "TOTAL error rate = 0.188869\n",
    "=== total 90800 match 15016 /// 4,7 => 1,5\n",
    "TOTAL 90800, precision 0.472030, recall 0.416280, f_score 0.442406\n",
    "TOTAL true = 17027\n",
    "TOTAL error rate = 0.196773\n",
    "=== total 91400 match 23705 /// 2,7 => 1,5\n",
    "TOTAL 91400, precision 0.453449, recall 0.622842, f_score 0.524815\n",
    "TOTAL true = 17258\n",
    "TOTAL error rate = 0.212965"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############1_2_val_3\n",
    "=== total 536900 match 120543 ## previously - not sure which params\n",
    "TOTAL 536900, precision 0.348216, recall 0.684915, f_score 0.461700\n",
    "TOTAL true = 61285\n",
    "TOTAL error rate = 0.182302\n",
    "=== total 538600 match 208051 /// 2,7 => 1,5\n",
    "TOTAL 538600, precision 0.279508, recall 0.944471, f_score 0.431359\n",
    "TOTAL true = 61571\n",
    "TOTAL error rate = 0.284660"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############1_3_val_2\n",
    "=== total 329000 match 111791 /// 2,7 => 1,5\n",
    "TOTAL 329000, precision 0.344285, recall 0.757131, f_score 0.473334\n",
    "TOTAL true = 50834\n",
    "TOTAL error rate = 0.260331"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T20:47:01.403854Z",
     "start_time": "2019-04-03T19:52:46.394114Z"
    }
   },
   "outputs": [],
   "source": [
    "#TRAIN 3\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model1.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T05:24:20.228262Z",
     "start_time": "2019-04-04T04:18:30.422157Z"
    }
   },
   "outputs": [],
   "source": [
    "# validate 2\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "  NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 5)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/tear_model2.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T05:53:22.125080Z",
     "start_time": "2019-04-04T05:34:40.304989Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model3.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}